[ 530.958681] env[64020]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'linux_bridge' {{(pid=64020) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 530.959111] env[64020]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'noop' {{(pid=64020) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 530.959111] env[64020]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'ovs' {{(pid=64020) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 530.959430] env[64020]: INFO os_vif [-] Loaded VIF plugins: linux_bridge, noop, ovs [ 531.046253] env[64020]: DEBUG oslo_concurrency.processutils [-] Running cmd (subprocess): grep -F node.session.scan /sbin/iscsiadm {{(pid=64020) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:390}} [ 531.055439] env[64020]: DEBUG oslo_concurrency.processutils [-] CMD "grep -F node.session.scan /sbin/iscsiadm" returned: 0 in 0.009s {{(pid=64020) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:428}} [ 531.097766] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ffbff08d-0e50-4414-862b-66f595893a06 None None] Creating reply queue: reply_57893177120949e6a93cb88e15cd42b4 [ 531.106265] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ffbff08d-0e50-4414-862b-66f595893a06 None None] Expecting reply to msg d5d6440574ed4b0a9ceeefe9b22be16f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 531.118990] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5d6440574ed4b0a9ceeefe9b22be16f [ 531.670089] env[64020]: INFO nova.virt.driver [None req-ffbff08d-0e50-4414-862b-66f595893a06 None None] Loading compute driver 'vmwareapi.VMwareVCDriver' [ 531.753629] env[64020]: DEBUG oslo_concurrency.lockutils [-] Acquiring lock "oslo_vmware_api_lock" by "oslo_vmware.api.VMwareAPISession._create_session" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 531.753847] env[64020]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" acquired by "oslo_vmware.api.VMwareAPISession._create_session" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 531.753919] env[64020]: DEBUG oslo_vmware.service [-] Creating suds client with soap_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk' and wsdl_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk/vimService.wsdl' {{(pid=64020) __init__ /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:242}} [ 534.712569] env[64020]: DEBUG oslo_vmware.service [-] Invoking ServiceInstance.RetrieveServiceContent with opID=oslo.vmware-055731aa-a88a-4cd1-b6c7-861b3d2e6c7a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.728276] env[64020]: DEBUG oslo_vmware.api [-] Logging into host: vc1.osci.c.eu-de-1.cloud.sap. {{(pid=64020) _create_session /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:242}} [ 534.728385] env[64020]: DEBUG oslo_vmware.service [-] Invoking SessionManager.Login with opID=oslo.vmware-9161be79-cd32-4a6d-bd63-b5f31e923673 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.753841] env[64020]: INFO oslo_vmware.api [-] Successfully established new session; session ID is b2672. [ 534.754004] env[64020]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" "released" by "oslo_vmware.api.VMwareAPISession._create_session" :: held 3.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 534.754508] env[64020]: INFO nova.virt.vmwareapi.driver [None req-ffbff08d-0e50-4414-862b-66f595893a06 None None] VMware vCenter version: 7.0.3 [ 534.757880] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b539b11c-3e81-497e-b46f-b2df4c22235b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.778386] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3eacff34-0018-457a-9229-c23a7fb2e132 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.784432] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4eaa5d24-6565-4427-8f66-3b55f0d83d6f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.791924] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8d8ed3c-b678-46c0-bb71-2969a1841a40 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.804929] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c2e1e10-b987-440f-aba7-07cc21d1a90b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.810737] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5fbb716-e167-45b7-8d35-904e71d0442e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.840459] env[64020]: DEBUG oslo_vmware.service [-] Invoking ExtensionManager.FindExtension with opID=oslo.vmware-d935aabc-babb-4e0e-982b-c53269fb403a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.845629] env[64020]: DEBUG nova.virt.vmwareapi.driver [None req-ffbff08d-0e50-4414-862b-66f595893a06 None None] Extension org.openstack.compute already exists. {{(pid=64020) _register_openstack_extension /opt/stack/nova/nova/virt/vmwareapi/driver.py:224}} [ 534.848296] env[64020]: INFO nova.compute.provider_config [None req-ffbff08d-0e50-4414-862b-66f595893a06 None None] No provider configs found in /etc/nova/provider_config/. If files are present, ensure the Nova process has access. [ 534.848978] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ffbff08d-0e50-4414-862b-66f595893a06 None None] Expecting reply to msg b09dc86bb85840b09f02fcc907b7f532 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 534.865125] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b09dc86bb85840b09f02fcc907b7f532 [ 535.353983] env[64020]: DEBUG nova.context [None req-ffbff08d-0e50-4414-862b-66f595893a06 None None] Found 2 cells: 00000000-0000-0000-0000-000000000000(cell0),68fb4695-dd42-413d-a96d-b3e3cfe20816(cell1) {{(pid=64020) load_cells /opt/stack/nova/nova/context.py:464}} [ 535.356082] env[64020]: DEBUG oslo_concurrency.lockutils [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.356334] env[64020]: DEBUG oslo_concurrency.lockutils [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.357032] env[64020]: DEBUG oslo_concurrency.lockutils [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.357473] env[64020]: DEBUG oslo_concurrency.lockutils [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Acquiring lock "68fb4695-dd42-413d-a96d-b3e3cfe20816" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.357661] env[64020]: DEBUG oslo_concurrency.lockutils [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Lock "68fb4695-dd42-413d-a96d-b3e3cfe20816" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.358678] env[64020]: DEBUG oslo_concurrency.lockutils [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Lock "68fb4695-dd42-413d-a96d-b3e3cfe20816" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.379334] env[64020]: INFO dbcounter [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Registered counter for database nova_cell0 [ 535.387154] env[64020]: INFO dbcounter [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Registered counter for database nova_cell1 [ 535.390310] env[64020]: DEBUG oslo_db.sqlalchemy.engines [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=64020) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 535.390650] env[64020]: DEBUG oslo_db.sqlalchemy.engines [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=64020) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 535.395511] env[64020]: ERROR nova.db.main.api [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 535.395511] env[64020]: result = function(*args, **kwargs) [ 535.395511] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 535.395511] env[64020]: return func(*args, **kwargs) [ 535.395511] env[64020]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 535.395511] env[64020]: result = fn(*args, **kwargs) [ 535.395511] env[64020]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 535.395511] env[64020]: return f(*args, **kwargs) [ 535.395511] env[64020]: File "/opt/stack/nova/nova/objects/service.py", line 553, in _db_service_get_minimum_version [ 535.395511] env[64020]: return db.service_get_minimum_version(context, binaries) [ 535.395511] env[64020]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 535.395511] env[64020]: _check_db_access() [ 535.395511] env[64020]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 535.395511] env[64020]: stacktrace = ''.join(traceback.format_stack()) [ 535.395511] env[64020]: [ 535.396213] env[64020]: ERROR nova.db.main.api [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 535.396213] env[64020]: result = function(*args, **kwargs) [ 535.396213] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 535.396213] env[64020]: return func(*args, **kwargs) [ 535.396213] env[64020]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 535.396213] env[64020]: result = fn(*args, **kwargs) [ 535.396213] env[64020]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 535.396213] env[64020]: return f(*args, **kwargs) [ 535.396213] env[64020]: File "/opt/stack/nova/nova/objects/service.py", line 553, in _db_service_get_minimum_version [ 535.396213] env[64020]: return db.service_get_minimum_version(context, binaries) [ 535.396213] env[64020]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 535.396213] env[64020]: _check_db_access() [ 535.396213] env[64020]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 535.396213] env[64020]: stacktrace = ''.join(traceback.format_stack()) [ 535.396213] env[64020]: [ 535.396568] env[64020]: WARNING nova.objects.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Failed to get minimum service version for cell 00000000-0000-0000-0000-000000000000 [ 535.396720] env[64020]: WARNING nova.objects.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Failed to get minimum service version for cell 68fb4695-dd42-413d-a96d-b3e3cfe20816 [ 535.397149] env[64020]: DEBUG oslo_concurrency.lockutils [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Acquiring lock "singleton_lock" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 535.397320] env[64020]: DEBUG oslo_concurrency.lockutils [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Acquired lock "singleton_lock" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 535.397562] env[64020]: DEBUG oslo_concurrency.lockutils [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Releasing lock "singleton_lock" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 535.397874] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Full set of CONF: {{(pid=64020) _wait_for_exit_or_signal /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:363}} [ 535.398014] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ******************************************************************************** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2600}} [ 535.398138] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] Configuration options gathered from: {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2601}} [ 535.398272] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] command line args: ['--config-file', '/etc/nova/nova.conf', '--config-file', '/etc/nova/nova-cpu-common.conf', '--config-file', '/etc/nova/nova-cpu-1.conf'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2602}} [ 535.398452] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] config files: ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2603}} [ 535.398575] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ================================================================================ {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2605}} [ 535.398771] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] allow_resize_to_same_host = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.398939] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] arq_binding_timeout = 300 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.399067] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] backdoor_port = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.399190] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] backdoor_socket = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.399352] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] block_device_allocate_retries = 60 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.399512] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] block_device_allocate_retries_interval = 3 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.399676] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cert = self.pem {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.399837] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute_driver = vmwareapi.VMwareVCDriver {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.400011] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute_monitors = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.400182] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] config_dir = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.400353] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] config_drive_format = iso9660 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.400483] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] config_file = ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.400644] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] config_source = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.400807] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] console_host = devstack {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.400979] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] control_exchange = nova {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.401427] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cpu_allocation_ratio = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.401632] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] daemon = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.401813] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] debug = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.401974] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] default_access_ip_network_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.402155] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] default_availability_zone = nova {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.402329] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] default_ephemeral_format = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.402491] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] default_green_pool_size = 1000 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.402732] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'glanceclient=WARN', 'oslo.privsep.daemon=INFO'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.402896] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] default_schedule_zone = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.403052] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] disk_allocation_ratio = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.403212] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] enable_new_services = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.403385] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] enabled_apis = ['osapi_compute'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.403544] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] enabled_ssl_apis = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.403701] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] flat_injected = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.403858] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] force_config_drive = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.404021] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] force_raw_images = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.404191] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] graceful_shutdown_timeout = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.404355] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] heal_instance_info_cache_interval = 60 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.404566] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] host = cpu-1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.404736] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] initial_cpu_allocation_ratio = 4.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.404896] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] initial_disk_allocation_ratio = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.405054] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] initial_ram_allocation_ratio = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.405278] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] injected_network_template = /opt/stack/nova/nova/virt/interfaces.template {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.405440] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] instance_build_timeout = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.405598] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] instance_delete_interval = 300 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.405758] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] instance_format = [instance: %(uuid)s] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.405919] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] instance_name_template = instance-%08x {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.406081] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] instance_usage_audit = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.406252] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] instance_usage_audit_period = month {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.406414] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] instance_uuid_format = [instance: %(uuid)s] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.406573] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] instances_path = /opt/stack/data/nova/instances {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.406737] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] internal_service_availability_zone = internal {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.406890] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] key = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.407086] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] live_migration_retry_count = 30 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.407217] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] log_color = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.407377] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] log_config_append = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.407540] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.407695] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] log_dir = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.407848] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] log_file = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.407972] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] log_options = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.408159] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] log_rotate_interval = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.408331] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] log_rotate_interval_type = days {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.408492] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] log_rotation_type = none {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.408617] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.408740] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.408903] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.409075] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.409205] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.409392] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] long_rpc_timeout = 1800 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.409527] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] max_concurrent_builds = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.409687] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] max_concurrent_live_migrations = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.409843] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] max_concurrent_snapshots = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.409998] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] max_local_block_devices = 3 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.410156] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] max_logfile_count = 30 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.410316] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] max_logfile_size_mb = 200 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.410471] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] maximum_instance_delete_attempts = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.410633] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] metadata_listen = 0.0.0.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.410800] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] metadata_listen_port = 8775 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.410966] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] metadata_workers = 2 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.411162] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] migrate_max_retries = -1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.411341] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] mkisofs_cmd = genisoimage {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.411583] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] my_block_storage_ip = 10.180.1.21 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.411732] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] my_ip = 10.180.1.21 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.411901] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] network_allocate_retries = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.412098] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] non_inheritable_image_properties = ['cache_in_nova', 'bittorrent'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.412282] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] osapi_compute_listen = 0.0.0.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.412442] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] osapi_compute_listen_port = 8774 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.412609] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] osapi_compute_unique_server_name_scope = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.412775] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] osapi_compute_workers = 2 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.412937] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] password_length = 12 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.413098] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] periodic_enable = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.413260] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] periodic_fuzzy_delay = 60 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.413470] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] pointer_model = usbtablet {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.413659] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] preallocate_images = none {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.413820] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] publish_errors = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.413947] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] pybasedir = /opt/stack/nova {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.414102] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ram_allocation_ratio = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.414262] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] rate_limit_burst = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.414432] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] rate_limit_except_level = CRITICAL {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.414584] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] rate_limit_interval = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.414739] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] reboot_timeout = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.414926] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] reclaim_instance_interval = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.415125] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] record = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.415303] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] reimage_timeout_per_gb = 60 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.415470] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] report_interval = 120 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.415630] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] rescue_timeout = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.415791] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] reserved_host_cpus = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.415950] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] reserved_host_disk_mb = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.416132] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] reserved_host_memory_mb = 512 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.416299] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] reserved_huge_pages = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.416458] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] resize_confirm_window = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.416615] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] resize_fs_using_block_device = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.416771] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] resume_guests_state_on_host_boot = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.416933] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] rootwrap_config = /etc/nova/rootwrap.conf {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.417091] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] rpc_response_timeout = 60 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.417251] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] run_external_periodic_tasks = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.417414] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] running_deleted_instance_action = reap {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.417572] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] running_deleted_instance_poll_interval = 1800 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.417727] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] running_deleted_instance_timeout = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.417883] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] scheduler_instance_sync_interval = 120 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.418048] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] service_down_time = 720 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.418214] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] servicegroup_driver = db {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.418369] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] shelved_offload_time = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.418525] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] shelved_poll_interval = 3600 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.418686] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] shutdown_timeout = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.418843] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] source_is_ipv6 = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.418996] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ssl_only = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.419237] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] state_path = /opt/stack/data/n-cpu-1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.419403] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] sync_power_state_interval = 600 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.419560] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] sync_power_state_pool_size = 1000 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.419720] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] syslog_log_facility = LOG_USER {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.419870] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] tempdir = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.420036] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] timeout_nbd = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.420208] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] transport_url = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.420364] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] update_resources_interval = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.420519] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] use_cow_images = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.420672] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] use_eventlog = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.420825] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] use_journal = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.420978] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] use_json = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.421161] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] use_rootwrap_daemon = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.421323] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] use_stderr = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.421478] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] use_syslog = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.421628] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vcpu_pin_set = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.421789] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plugging_is_fatal = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.421959] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plugging_timeout = 300 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.422133] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] virt_mkfs = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.422282] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] volume_usage_poll_interval = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.422438] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] watch_log_file = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.422599] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] web = /usr/share/spice-html5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.422774] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_concurrency.disable_process_locking = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.423344] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_concurrency.lock_path = /opt/stack/data/n-cpu-1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.423585] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.423769] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_metrics.metrics_enabled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.423943] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_metrics.metrics_process_name = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.424128] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.424299] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.424481] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.auth_strategy = keystone {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.424646] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.compute_link_prefix = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.424819] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.424990] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.dhcp_domain = novalocal {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.425156] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.enable_instance_password = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.425322] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.glance_link_prefix = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.425485] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.instance_list_cells_batch_fixed_size = 100 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.425655] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.instance_list_cells_batch_strategy = distributed {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.425815] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.instance_list_per_project_cells = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.426035] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.list_records_by_skipping_down_cells = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.426205] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.local_metadata_per_cell = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.426374] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.max_limit = 1000 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.426542] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.metadata_cache_expiration = 15 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.426715] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.neutron_default_tenant_id = default {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.426883] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.use_neutron_default_nets = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.427054] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.vendordata_dynamic_connect_timeout = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.427220] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.vendordata_dynamic_failure_fatal = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.427386] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.vendordata_dynamic_read_timeout = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.427557] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.vendordata_dynamic_ssl_certfile = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.427730] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.vendordata_dynamic_targets = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.427895] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.vendordata_jsonfile_path = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.428088] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api.vendordata_providers = ['StaticJSON'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.428290] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.backend = dogpile.cache.memcached {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.428462] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.backend_argument = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.428637] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.config_prefix = cache.oslo {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.428806] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.dead_timeout = 60.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.428971] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.debug_cache_backend = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.429133] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.enable_retry_client = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.429296] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.enable_socket_keepalive = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.429463] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.enabled = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.429623] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.enforce_fips_mode = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.429789] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.expiration_time = 600 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.429955] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.hashclient_retry_attempts = 2 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.430120] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.hashclient_retry_delay = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.430287] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.memcache_dead_retry = 300 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.430447] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.memcache_password = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.430609] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.memcache_pool_connection_get_timeout = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.430770] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.memcache_pool_flush_on_reconnect = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.430934] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.memcache_pool_maxsize = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.431129] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.memcache_pool_unused_timeout = 60 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.431297] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.memcache_sasl_enabled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.431476] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.memcache_servers = ['localhost:11211'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.431640] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.memcache_socket_timeout = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.431800] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.memcache_username = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.431961] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.proxies = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.432148] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.redis_db = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.432311] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.redis_password = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.432886] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.redis_sentinel_service_name = mymaster {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.433219] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.redis_sentinels = ['localhost:26379'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.433443] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.redis_server = localhost:6379 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.433630] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.redis_socket_timeout = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.433795] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.redis_username = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.433961] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.retry_attempts = 2 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.434127] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.retry_delay = 0.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.434293] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.socket_keepalive_count = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.434454] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.socket_keepalive_idle = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.434615] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.socket_keepalive_interval = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.434773] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.tls_allowed_ciphers = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.434928] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.tls_cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.435082] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.tls_certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.435243] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.tls_enabled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.435398] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cache.tls_keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.435565] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.auth_section = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.435737] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.auth_type = password {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.435895] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.436085] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.catalog_info = volumev3::publicURL {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.436250] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.436410] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.436571] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.cross_az_attach = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.436731] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.debug = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.436888] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.endpoint_template = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.437050] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.http_retries = 3 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.437213] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.437369] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.437535] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.os_region_name = RegionOne {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.437696] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.437904] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cinder.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.438023] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.consecutive_build_service_disable_threshold = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.438183] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.cpu_dedicated_set = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.438340] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.cpu_shared_set = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.438497] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.image_type_exclude_list = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.438655] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.live_migration_wait_for_vif_plug = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.438818] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.max_concurrent_disk_ops = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.438980] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.max_disk_devices_to_attach = -1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.439140] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.packing_host_numa_cells_allocation_strategy = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.439307] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.provider_config_location = /etc/nova/provider_config/ {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.439470] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.resource_provider_association_refresh = 300 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.439628] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.sharing_providers_max_uuids_per_request = 200 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.439789] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.shutdown_retry_interval = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.439966] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] compute.vmdk_allowed_types = ['streamOptimized', 'monolithicSparse'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.440158] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] conductor.workers = 2 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.440337] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] console.allowed_origins = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.440496] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] console.ssl_ciphers = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.440661] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] console.ssl_minimum_version = default {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.440827] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] consoleauth.enforce_session_timeout = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.440995] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] consoleauth.token_ttl = 600 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.441184] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.441345] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.441506] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.441664] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.connect_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.441819] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.connect_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.441975] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.endpoint_override = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.442133] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.442289] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.442446] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.max_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.442600] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.min_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.442754] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.region_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.442908] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.retriable_status_codes = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.443062] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.service_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.443229] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.service_type = accelerator {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.443405] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.443573] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.status_code_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.443728] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.status_code_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.443884] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.444076] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.valid_interfaces = ['internal', 'public'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.444243] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] cyborg.version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.444421] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.backend = sqlalchemy {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.444587] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.connection = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.444753] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.connection_debug = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.444919] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.connection_parameters = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.445085] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.connection_recycle_time = 3600 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.445250] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.connection_trace = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.445410] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.db_inc_retry_interval = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.445572] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.db_max_retries = 20 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.445732] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.db_max_retry_interval = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.445892] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.db_retry_interval = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.446050] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.max_overflow = 50 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.446211] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.max_pool_size = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.446373] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.max_retries = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.446533] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.mysql_sql_mode = TRADITIONAL {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.446687] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.mysql_wsrep_sync_wait = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.446841] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.pool_timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.446999] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.retry_interval = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.447154] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.slave_connection = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.447314] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.sqlite_synchronous = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.447471] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] database.use_db_reconnect = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.447643] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.backend = sqlalchemy {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.447807] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.connection = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.448072] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.connection_debug = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.448141] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.connection_parameters = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.448304] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.connection_recycle_time = 3600 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.448464] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.connection_trace = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.448624] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.db_inc_retry_interval = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.448784] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.db_max_retries = 20 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.448946] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.db_max_retry_interval = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.449106] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.db_retry_interval = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.449268] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.max_overflow = 50 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.449426] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.max_pool_size = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.449582] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.max_retries = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.449746] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.mysql_sql_mode = TRADITIONAL {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.449901] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.mysql_wsrep_sync_wait = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.450055] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.pool_timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.450215] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.retry_interval = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.450369] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.slave_connection = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.450527] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] api_database.sqlite_synchronous = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.450699] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] devices.enabled_mdev_types = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.450872] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ephemeral_storage_encryption.cipher = aes-xts-plain64 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.451055] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ephemeral_storage_encryption.default_format = luks {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.451234] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ephemeral_storage_encryption.enabled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.451401] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ephemeral_storage_encryption.key_size = 512 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.451569] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.api_servers = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.451729] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.451884] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.452086] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.452290] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.connect_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.452455] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.connect_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.452616] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.debug = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.452778] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.default_trusted_certificate_ids = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.452938] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.enable_certificate_validation = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.453097] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.enable_rbd_download = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.453255] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.endpoint_override = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.453441] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.453614] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.453772] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.max_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.453926] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.min_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.454086] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.num_retries = 3 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.454255] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.rbd_ceph_conf = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.454416] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.rbd_connect_timeout = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.454580] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.rbd_pool = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.454744] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.rbd_user = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.454899] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.region_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.455052] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.retriable_status_codes = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.455207] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.service_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.455370] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.service_type = image {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.455527] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.455681] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.status_code_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.455836] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.status_code_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.455990] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.456212] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.valid_interfaces = ['internal', 'public'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.456433] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.verify_glance_signatures = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.456610] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] glance.version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.456778] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] guestfs.debug = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.456945] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] mks.enabled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.457306] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] mks.mksproxy_base_url = http://127.0.0.1:6090/ {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.457499] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] image_cache.manager_interval = 2400 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.457669] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] image_cache.precache_concurrency = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.457838] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] image_cache.remove_unused_base_images = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.458007] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] image_cache.remove_unused_original_minimum_age_seconds = 86400 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.458174] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] image_cache.remove_unused_resized_minimum_age_seconds = 3600 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.458350] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] image_cache.subdirectory_name = _base {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.458527] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.api_max_retries = 60 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.458692] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.api_retry_interval = 2 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.458852] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.auth_section = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.459014] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.auth_type = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.459170] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.459323] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.459481] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.459638] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.conductor_group = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.459792] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.connect_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.459948] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.connect_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.460120] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.endpoint_override = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.460286] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.460443] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.460598] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.max_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.460750] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.min_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.460915] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.peer_list = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.461100] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.region_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.461277] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.retriable_status_codes = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.461445] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.serial_console_state_timeout = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.461603] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.service_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.461769] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.service_type = baremetal {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.461928] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.shard = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.462113] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.462284] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.status_code_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.462443] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.status_code_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.462600] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.462781] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.valid_interfaces = ['internal', 'public'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.462942] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ironic.version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.463123] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] key_manager.backend = nova.keymgr.conf_key_mgr.ConfKeyManager {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.463295] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] key_manager.fixed_key = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.463501] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.auth_endpoint = http://localhost/identity/v3 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.463668] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.barbican_api_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.463824] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.barbican_endpoint = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.463991] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.barbican_endpoint_type = public {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.464188] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.barbican_region_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.464361] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.464521] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.464682] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.464841] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.464995] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.465162] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.number_of_retries = 60 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.465324] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.retry_delay = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.465485] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.send_service_user_token = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.465639] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.465795] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.465954] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.verify_ssl = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.466109] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican.verify_ssl_path = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.466278] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican_service_user.auth_section = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.466438] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican_service_user.auth_type = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.466592] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican_service_user.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.466745] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican_service_user.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.466903] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican_service_user.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.467116] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican_service_user.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.467299] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican_service_user.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.467464] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican_service_user.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.467625] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] barbican_service_user.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.467789] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vault.approle_role_id = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.467948] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vault.approle_secret_id = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.468188] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vault.kv_mountpoint = secret {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.468301] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vault.kv_path = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.468466] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vault.kv_version = 2 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.468622] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vault.namespace = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.468777] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vault.root_token_id = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.468931] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vault.ssl_ca_crt_file = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.469093] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vault.timeout = 60.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.469254] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vault.use_ssl = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.469422] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vault.vault_url = http://127.0.0.1:8200 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.469591] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.auth_section = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.469750] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.auth_type = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.469907] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.470062] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.470222] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.470376] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.connect_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.470529] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.connect_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.470681] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.endpoint_override = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.470835] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.470987] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.471164] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.max_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.471323] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.min_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.471474] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.region_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.471630] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.retriable_status_codes = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.471781] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.service_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.471945] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.service_type = identity {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.472142] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.472316] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.status_code_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.472473] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.status_code_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.472630] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.472810] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.valid_interfaces = ['internal', 'public'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.472969] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] keystone.version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.473170] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.connection_uri = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.473333] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.cpu_mode = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.473524] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.cpu_model_extra_flags = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.473695] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.cpu_models = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.473863] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.cpu_power_governor_high = performance {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.474029] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.cpu_power_governor_low = powersave {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.474190] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.cpu_power_management = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.474356] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.cpu_power_management_strategy = cpu_state {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.474519] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.device_detach_attempts = 8 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.474681] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.device_detach_timeout = 20 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.474845] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.disk_cachemodes = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.475002] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.disk_prefix = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.475160] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.enabled_perf_events = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.475323] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.file_backed_memory = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.475485] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.gid_maps = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.475639] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.hw_disk_discard = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.475793] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.hw_machine_type = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.475960] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.images_rbd_ceph_conf = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.476165] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.images_rbd_glance_copy_poll_interval = 15 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.476352] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.images_rbd_glance_copy_timeout = 600 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.476523] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.images_rbd_glance_store_name = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.476688] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.images_rbd_pool = rbd {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.476853] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.images_type = default {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.477009] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.images_volume_group = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.477170] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.inject_key = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.477335] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.inject_partition = -2 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.477494] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.inject_password = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.477657] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.iscsi_iface = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.477814] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.iser_use_multipath = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.477976] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_bandwidth = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.478137] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_completion_timeout = 800 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.478300] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_downtime = 500 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.478462] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_downtime_delay = 75 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.478623] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_downtime_steps = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.478793] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_inbound_addr = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.478957] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_permit_auto_converge = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.479114] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_permit_post_copy = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.479271] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_scheme = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.479444] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_timeout_action = abort {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.479606] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_tunnelled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.479765] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_uri = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.479924] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.live_migration_with_native_tls = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.480143] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.max_queues = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.480260] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.mem_stats_period_seconds = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.480547] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.migration_inbound_addr = 10.180.1.21 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.480742] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.nfs_mount_options = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.481072] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.nfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.481251] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.num_aoe_discover_tries = 3 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.481420] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.num_iser_scan_tries = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.481584] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.num_memory_encrypted_guests = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.481748] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.num_nvme_discover_tries = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.481912] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.num_pcie_ports = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.482096] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.num_volume_scan_tries = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.482279] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.pmem_namespaces = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.482439] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.quobyte_client_cfg = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.482720] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.quobyte_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.482893] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.rbd_connect_timeout = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.483060] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.rbd_destroy_volume_retries = 12 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.483227] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.rbd_destroy_volume_retry_interval = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.483404] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.rbd_secret_uuid = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.483574] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.rbd_user = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.483737] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.realtime_scheduler_priority = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.483906] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.remote_filesystem_transport = ssh {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.484079] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.rescue_image_id = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.484245] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.rescue_kernel_id = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.484399] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.rescue_ramdisk_id = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.484569] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.rng_dev_path = /dev/urandom {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.484726] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.rx_queue_size = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.484891] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.smbfs_mount_options = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.485170] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.smbfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.485354] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.snapshot_compression = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.485515] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.snapshot_image_format = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.485730] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.snapshots_directory = /opt/stack/data/nova/instances/snapshots {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.485896] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.sparse_logical_volumes = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.486056] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.swtpm_enabled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.486226] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.swtpm_group = tss {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.486391] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.swtpm_user = tss {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.486554] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.sysinfo_serial = unique {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.486711] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.tb_cache_size = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.486866] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.tx_queue_size = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.487027] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.uid_maps = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.487188] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.use_virtio_for_bridges = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.487356] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.virt_type = kvm {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.487565] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.volume_clear = zero {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.487804] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.volume_clear_size = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.487991] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.volume_use_multipath = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.488185] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.vzstorage_cache_path = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.488384] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.vzstorage_log_path = /var/log/vstorage/%(cluster_name)s/nova.log.gz {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.488546] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.vzstorage_mount_group = qemu {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.488713] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.vzstorage_mount_opts = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.488880] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.vzstorage_mount_perms = 0770 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.489151] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.vzstorage_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.489328] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.vzstorage_mount_user = stack {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.489493] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] libvirt.wait_soft_reboot_seconds = 120 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.489666] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.auth_section = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.489836] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.auth_type = password {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.489996] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.490153] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.490315] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.490469] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.connect_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.490635] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.connect_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.490859] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.default_floating_pool = public {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.491046] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.endpoint_override = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.491218] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.extension_sync_interval = 600 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.491383] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.http_retries = 3 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.491542] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.491699] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.491854] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.max_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.492040] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.metadata_proxy_shared_secret = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.492220] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.min_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.492389] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.ovs_bridge = br-int {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.492552] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.physnets = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.492719] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.region_name = RegionOne {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.492875] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.retriable_status_codes = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.493040] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.service_metadata_proxy = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.493202] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.service_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.493382] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.service_type = network {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.493559] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.493717] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.status_code_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.493874] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.status_code_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.494029] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.494208] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.valid_interfaces = ['internal', 'public'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.494369] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] neutron.version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.494538] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] notifications.bdms_in_notifications = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.494711] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] notifications.default_level = INFO {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.494881] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] notifications.notification_format = unversioned {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.495042] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] notifications.notify_on_state_change = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.495217] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] notifications.versioned_notifications_topics = ['versioned_notifications'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.495396] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] pci.alias = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.495560] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] pci.device_spec = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.495721] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] pci.report_in_placement = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.495889] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.auth_section = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.496074] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.auth_type = password {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.496246] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.auth_url = http://10.180.1.21/identity {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.496405] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.496562] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.496722] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.496878] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.connect_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.497034] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.connect_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.497192] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.default_domain_id = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.497346] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.default_domain_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.497502] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.domain_id = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.497659] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.domain_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.497813] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.endpoint_override = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.497973] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.498128] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.498284] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.max_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.498482] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.min_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.498600] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.password = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.498755] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.project_domain_id = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.498940] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.project_domain_name = Default {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.499082] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.project_id = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.499253] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.project_name = service {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.499417] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.region_name = RegionOne {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.499575] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.retriable_status_codes = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.499730] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.service_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.499895] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.service_type = placement {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.500067] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.500249] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.status_code_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.500414] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.status_code_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.500573] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.system_scope = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.500732] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.500890] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.trust_id = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.501067] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.user_domain_id = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.501242] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.user_domain_name = Default {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.501408] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.user_id = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.501581] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.username = nova {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.501761] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.valid_interfaces = ['internal', 'public'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.501923] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] placement.version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.502101] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.cores = 20 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.502265] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.count_usage_from_placement = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.502436] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.driver = nova.quota.DbQuotaDriver {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.502605] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.injected_file_content_bytes = 10240 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.502771] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.injected_file_path_length = 255 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.502936] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.injected_files = 5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.503103] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.instances = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.503269] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.key_pairs = 100 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.503461] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.metadata_items = 128 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.503640] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.ram = 51200 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.503804] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.recheck_quota = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.503970] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.server_group_members = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.504154] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] quota.server_groups = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.504330] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] scheduler.discover_hosts_in_cells_interval = -1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.504493] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] scheduler.enable_isolated_aggregate_filtering = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.504651] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] scheduler.image_metadata_prefilter = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.504810] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] scheduler.limit_tenants_to_placement_aggregate = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.504974] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] scheduler.max_attempts = 3 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.505148] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] scheduler.max_placement_results = 1000 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.505335] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] scheduler.placement_aggregate_required_for_tenants = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.505500] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] scheduler.query_placement_for_image_type_support = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.505660] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] scheduler.query_placement_for_routed_network_aggregates = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.505832] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] scheduler.workers = 2 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.506003] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.aggregate_image_properties_isolation_namespace = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.506171] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.aggregate_image_properties_isolation_separator = . {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.506351] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.available_filters = ['nova.scheduler.filters.all_filters'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.506518] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.build_failure_weight_multiplier = 1000000.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.506679] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.cpu_weight_multiplier = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.506842] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.cross_cell_move_weight_multiplier = 1000000.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.507003] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.disk_weight_multiplier = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.507189] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.enabled_filters = ['ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'SameHostFilter', 'DifferentHostFilter'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.507357] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.host_subset_size = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.507519] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.hypervisor_version_weight_multiplier = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.507676] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.image_properties_default_architecture = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.507834] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.io_ops_weight_multiplier = -1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.507994] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.isolated_hosts = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.508200] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.isolated_images = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.508384] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.max_instances_per_host = 50 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.508582] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.max_io_ops_per_host = 8 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.508792] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.num_instances_weight_multiplier = 0.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.509003] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.pci_in_placement = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.509121] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.pci_weight_multiplier = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.509285] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.ram_weight_multiplier = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.509446] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.restrict_isolated_hosts_to_isolated_images = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.509606] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.shuffle_best_same_weighed_hosts = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.509767] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.soft_affinity_weight_multiplier = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.509927] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.soft_anti_affinity_weight_multiplier = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.510085] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.track_instance_changes = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.510261] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] filter_scheduler.weight_classes = ['nova.scheduler.weights.all_weighers'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.510460] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] metrics.required = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.510590] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] metrics.weight_multiplier = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.510751] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] metrics.weight_of_unavailable = -10000.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.510913] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] metrics.weight_setting = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.511253] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] serial_console.base_url = ws://127.0.0.1:6083/ {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.511433] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] serial_console.enabled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.511611] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] serial_console.port_range = 10000:20000 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.511782] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] serial_console.proxyclient_address = 127.0.0.1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.511949] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] serial_console.serialproxy_host = 0.0.0.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.512134] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] serial_console.serialproxy_port = 6083 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.512306] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] service_user.auth_section = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.512478] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] service_user.auth_type = password {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.512636] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] service_user.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.512790] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] service_user.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.512948] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] service_user.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.513105] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] service_user.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.513261] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] service_user.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.513450] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] service_user.send_service_user_token = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.513621] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] service_user.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.513782] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] service_user.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.513949] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.agent_enabled = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.514109] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.enabled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.514413] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.html5proxy_base_url = http://127.0.0.1:6082/spice_auto.html {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.514599] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.html5proxy_host = 0.0.0.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.514781] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.html5proxy_port = 6082 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.514950] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.image_compression = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.515108] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.jpeg_compression = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.515266] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.playback_compression = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.515430] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.server_listen = 127.0.0.1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.515594] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.server_proxyclient_address = 127.0.0.1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.515750] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.streaming_mode = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.515902] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] spice.zlib_compression = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.516077] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] upgrade_levels.baseapi = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.516251] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] upgrade_levels.compute = auto {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.516407] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] upgrade_levels.conductor = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.516563] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] upgrade_levels.scheduler = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.516726] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vendordata_dynamic_auth.auth_section = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.516887] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vendordata_dynamic_auth.auth_type = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.517042] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vendordata_dynamic_auth.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.517198] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vendordata_dynamic_auth.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.517357] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vendordata_dynamic_auth.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.517513] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vendordata_dynamic_auth.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.517666] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vendordata_dynamic_auth.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.517823] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vendordata_dynamic_auth.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.517977] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vendordata_dynamic_auth.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.518147] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.api_retry_count = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.518308] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.ca_file = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.518474] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.cache_prefix = devstack-image-cache {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.518679] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.cluster_name = testcl1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.518797] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.connection_pool_size = 10 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.518954] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.console_delay_seconds = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.519120] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.datastore_regex = ^datastore.* {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.519319] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.host_ip = vc1.osci.c.eu-de-1.cloud.sap {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.519488] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.host_password = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.519652] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.host_port = 443 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.519818] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.host_username = administrator@vsphere.local {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.519983] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.insecure = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.520157] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.integration_bridge = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.520322] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.maximum_objects = 100 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.520478] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.pbm_default_policy = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.520637] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.pbm_enabled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.520792] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.pbm_wsdl_location = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.520953] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.serial_log_dir = /opt/vmware/vspc {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.521142] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.serial_port_proxy_uri = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.521306] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.serial_port_service_uri = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.521471] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.task_poll_interval = 0.5 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.521639] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.use_linked_clone = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.521804] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.vnc_keymap = en-us {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.521966] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.vnc_port = 5900 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.522129] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vmware.vnc_port_total = 10000 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.522311] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vnc.auth_schemes = ['none'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.522480] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vnc.enabled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.522776] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vnc.novncproxy_base_url = http://127.0.0.1:6080/vnc_auto.html {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.522960] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vnc.novncproxy_host = 0.0.0.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.523128] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vnc.novncproxy_port = 6080 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.523303] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vnc.server_listen = 127.0.0.1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.523507] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vnc.server_proxyclient_address = 127.0.0.1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.523671] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vnc.vencrypt_ca_certs = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.523830] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vnc.vencrypt_client_cert = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.523984] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vnc.vencrypt_client_key = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.524174] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.disable_compute_service_check_for_ffu = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.524339] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.disable_deep_image_inspection = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.524498] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.disable_fallback_pcpu_query = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.524657] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.disable_group_policy_check_upcall = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.524812] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.disable_libvirt_livesnapshot = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.524972] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.disable_rootwrap = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.525130] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.enable_numa_live_migration = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.525291] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.enable_qemu_monitor_announce_self = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.525448] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.ensure_libvirt_rbd_instance_dir_cleanup = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.525604] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.handle_virt_lifecycle_events = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.525763] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.libvirt_disable_apic = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.525917] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.never_download_image_if_on_rbd = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.526078] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.qemu_monitor_announce_self_count = 3 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.526239] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.qemu_monitor_announce_self_interval = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.526396] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.reserve_disk_resource_for_image_cache = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.526553] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.skip_cpu_compare_at_startup = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.526708] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.skip_cpu_compare_on_dest = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.526891] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.skip_hypervisor_version_check_on_lm = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.527074] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.skip_reserve_in_use_ironic_nodes = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.527237] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.unified_limits_count_pcpu_as_vcpu = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.527396] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.unified_limits_permissive_enforcement = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.527558] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] workarounds.wait_for_vif_plugged_event_during_hard_reboot = [] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.527740] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] wsgi.api_paste_config = /etc/nova/api-paste.ini {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.527909] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] wsgi.client_socket_timeout = 900 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.528087] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] wsgi.default_pool_size = 1000 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.528258] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] wsgi.keep_alive = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.528422] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] wsgi.max_header_line = 16384 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.528581] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] wsgi.secure_proxy_ssl_header = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.528776] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] wsgi.ssl_ca_file = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.528891] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] wsgi.ssl_cert_file = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.529061] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] wsgi.ssl_key_file = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.529243] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] wsgi.tcp_keepidle = 600 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.529421] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] wsgi.wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.529584] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] zvm.ca_file = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.529743] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] zvm.cloud_connector_url = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.530023] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] zvm.image_tmp_path = /opt/stack/data/n-cpu-1/images {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.530197] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] zvm.reachable_timeout = 300 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.530377] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_policy.enforce_new_defaults = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.530544] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_policy.enforce_scope = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.530718] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_policy.policy_default_rule = default {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.530894] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_policy.policy_dirs = ['policy.d'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.531088] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_policy.policy_file = policy.yaml {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.531268] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_policy.remote_content_type = application/x-www-form-urlencoded {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.531430] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_policy.remote_ssl_ca_crt_file = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.531586] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_policy.remote_ssl_client_crt_file = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.531742] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_policy.remote_ssl_client_key_file = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.531901] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_policy.remote_ssl_verify_server_crt = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.532094] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.532254] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.532430] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler.connection_string = messaging:// {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.532594] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler.enabled = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.532759] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler.es_doc_type = notification {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.532921] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler.es_scroll_size = 10000 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.533088] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler.es_scroll_time = 2m {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.533252] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler.filter_error_trace = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.533445] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler.hmac_keys = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.533617] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler.sentinel_service_name = mymaster {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.533782] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler.socket_timeout = 0.1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.533943] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler.trace_requests = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.534100] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler.trace_sqlalchemy = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.534275] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler_jaeger.process_tags = {} {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.534434] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler_jaeger.service_name_prefix = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.534594] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] profiler_otlp.service_name_prefix = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.534756] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] remote_debug.host = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.534913] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] remote_debug.port = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.535094] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.535258] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.535420] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.535582] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.535742] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.535901] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.536071] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.heartbeat_in_pthread = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.536239] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.heartbeat_rate = 3 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.536400] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.heartbeat_timeout_threshold = 60 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.536564] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.hostname = devstack {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.536718] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.kombu_compression = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.536882] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.537046] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.537209] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.537374] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.processname = nova-compute {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.537536] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.537695] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.537864] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.538024] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.538183] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_quorum_delivery_limit = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.538345] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_bytes = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.538508] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_length = 0 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.538666] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.538874] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.538984] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.539173] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_stream_fanout = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.539345] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.539506] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rabbit_transient_quorum_queue = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.539672] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.539837] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.ssl = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.540014] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.ssl_ca_file = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.540190] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.ssl_cert_file = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.540353] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.540519] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.ssl_key_file = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.540685] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.ssl_version = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.540845] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_rabbit.use_queue_manager = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.541059] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_notifications.driver = ['messagingv2'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.541215] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_notifications.retry = -1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.541402] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_notifications.topics = ['notifications'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.541575] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_messaging_notifications.transport_url = **** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.541747] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.auth_section = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.541910] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.auth_type = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.542068] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.cafile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.542223] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.certfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.542382] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.collect_timing = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.542538] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.connect_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.542692] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.connect_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.542848] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.endpoint_id = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.543000] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.endpoint_override = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.543159] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.insecure = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.543315] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.keyfile = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.543470] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.max_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.543626] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.min_version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.543782] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.region_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.543934] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.retriable_status_codes = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.544100] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.service_name = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.544262] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.service_type = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.544421] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.split_loggers = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.544576] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.status_code_retries = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.544731] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.status_code_retry_delay = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.544886] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.timeout = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.545044] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.valid_interfaces = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.545199] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_limit.version = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.545363] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_reports.file_event_handler = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.545529] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_reports.file_event_handler_interval = 1 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.545688] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] oslo_reports.log_dir = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.545858] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_linux_bridge_privileged.capabilities = [12] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.546019] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_linux_bridge_privileged.group = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.546177] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_linux_bridge_privileged.helper_command = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.546341] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_linux_bridge_privileged.logger_name = oslo_privsep.daemon {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.546504] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_linux_bridge_privileged.thread_pool_size = 8 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.546662] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_linux_bridge_privileged.user = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.546829] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_ovs_privileged.capabilities = [12, 1] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.546989] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_ovs_privileged.group = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.547145] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_ovs_privileged.helper_command = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.547308] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_ovs_privileged.logger_name = oslo_privsep.daemon {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.547469] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_ovs_privileged.thread_pool_size = 8 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.547624] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] vif_plug_ovs_privileged.user = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.547792] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_linux_bridge.flat_interface = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.547967] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_linux_bridge.forward_bridge_interface = ['all'] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.548152] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_linux_bridge.iptables_bottom_regex = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.548325] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_linux_bridge.iptables_drop_action = DROP {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.548491] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_linux_bridge.iptables_top_regex = {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.548654] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_linux_bridge.network_device_mtu = 1500 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.548816] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_linux_bridge.use_ipv6 = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.548972] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_linux_bridge.vlan_interface = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.549167] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_ovs.default_qos_type = linux-noop {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.549376] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_ovs.isolate_vif = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.549504] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_ovs.network_device_mtu = 1500 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.549668] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_ovs.ovs_vsctl_timeout = 120 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.549835] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_ovs.ovsdb_connection = tcp:127.0.0.1:6640 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.550002] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_ovs.ovsdb_interface = native {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.550159] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_vif_ovs.per_port_bridge = False {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.550323] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] os_brick.lock_path = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.550490] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] privsep_osbrick.capabilities = [21] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.550646] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] privsep_osbrick.group = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.550801] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] privsep_osbrick.helper_command = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.550962] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] privsep_osbrick.logger_name = os_brick.privileged {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.551150] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] privsep_osbrick.thread_pool_size = 8 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.551314] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] privsep_osbrick.user = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.551485] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] nova_sys_admin.capabilities = [0, 1, 2, 3, 12, 21] {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.551641] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] nova_sys_admin.group = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.551793] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] nova_sys_admin.helper_command = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.551955] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] nova_sys_admin.logger_name = oslo_privsep.daemon {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.552145] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] nova_sys_admin.thread_pool_size = 8 {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.552321] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] nova_sys_admin.user = None {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.552453] env[64020]: DEBUG oslo_service.service [None req-65ab6af8-e05f-4aad-ad2a-b21e155119b3 None None] ******************************************************************************** {{(pid=64020) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2624}} [ 535.552869] env[64020]: INFO nova.service [-] Starting compute node (version 0.1.0) [ 535.553725] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg e2048ad98e2b41ab949fc98abc8e2187 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 535.562198] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e2048ad98e2b41ab949fc98abc8e2187 [ 536.056693] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Getting list of instances from cluster (obj){ [ 536.056693] env[64020]: value = "domain-c8" [ 536.056693] env[64020]: _type = "ClusterComputeResource" [ 536.056693] env[64020]: } {{(pid=64020) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 536.057908] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca8196be-802e-424f-bd6a-5f0920238e5b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 536.067917] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Got total of 0 instances {{(pid=64020) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 536.068535] env[64020]: WARNING nova.virt.vmwareapi.driver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] The vmwareapi driver is not tested by the OpenStack project nor does it have clear maintainer(s) and thus its quality can not be ensured. It should be considered experimental and may be removed in a future release. If you are using the driver in production please let us know via the openstack-discuss mailing list. [ 536.069026] env[64020]: INFO nova.virt.node [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Generated node identity 092888c9-2221-4dfc-9104-eeeb335c764f [ 536.069269] env[64020]: INFO nova.virt.node [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Wrote node identity 092888c9-2221-4dfc-9104-eeeb335c764f to /opt/stack/data/n-cpu-1/compute_id [ 536.069711] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg ca654a73dc8f4f09bb53898a591663d1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 536.082039] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca654a73dc8f4f09bb53898a591663d1 [ 536.572467] env[64020]: WARNING nova.compute.manager [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Compute nodes ['092888c9-2221-4dfc-9104-eeeb335c764f'] for host cpu-1 were not found in the database. If this is the first time this service is starting on this host, then you can ignore this warning. [ 536.573197] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg 953f1c107d8645ec83e948a9f5908b9d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 536.599850] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 953f1c107d8645ec83e948a9f5908b9d [ 537.076503] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg 10f8e76df93e4789afe8dbf89b3c8078 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 537.088271] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10f8e76df93e4789afe8dbf89b3c8078 [ 537.579513] env[64020]: INFO nova.compute.manager [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Looking for unclaimed instances stuck in BUILDING status for nodes managed by this host [ 537.580019] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg 7612373ad5c44b89a886693f71cafbbd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 537.590334] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7612373ad5c44b89a886693f71cafbbd [ 538.082875] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg 19179648614c4e049e5c04d981d71f1f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 538.095043] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19179648614c4e049e5c04d981d71f1f [ 538.586351] env[64020]: WARNING nova.compute.manager [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] No compute node record found for host cpu-1. If this is the first time this service is starting on this host, then you can ignore this warning.: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host cpu-1 could not be found. [ 538.586633] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 538.586858] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 538.587008] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 538.587161] env[64020]: DEBUG nova.compute.resource_tracker [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=64020) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 538.588017] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80d7a998-bebc-4a7c-9c85-d8fddc5df56d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.596426] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e9248e92-8be5-4bd2-8cb4-83abba457f59 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.609960] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b371d63-cdca-4050-911c-aba986ff42ed {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.616420] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11983197-a7cf-4df5-b735-5c3937645150 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.644637] env[64020]: DEBUG nova.compute.resource_tracker [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181798MB free_disk=120GB free_vcpus=48 pci_devices=None {{(pid=64020) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 538.644765] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 538.644956] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 538.645288] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg 98baa8b800fa4e48aa00cf7680160a08 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 538.656524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98baa8b800fa4e48aa00cf7680160a08 [ 539.147516] env[64020]: WARNING nova.compute.resource_tracker [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] No compute node record for cpu-1:092888c9-2221-4dfc-9104-eeeb335c764f: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host 092888c9-2221-4dfc-9104-eeeb335c764f could not be found. [ 539.148716] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg 2f3fe580a8fd44aa9c69f70f070a4837 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 539.160548] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f3fe580a8fd44aa9c69f70f070a4837 [ 539.651651] env[64020]: INFO nova.compute.resource_tracker [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Compute node record created for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 with uuid: 092888c9-2221-4dfc-9104-eeeb335c764f [ 539.652123] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg fa81e1107e92419782118c7906723365 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 539.662729] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fa81e1107e92419782118c7906723365 [ 540.155278] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg 51d455dce9074e9c97ad67d15c40ab5b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 540.175070] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51d455dce9074e9c97ad67d15c40ab5b [ 540.657449] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg f1880ccccc544ac0b01073b9540618e4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 540.679097] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f1880ccccc544ac0b01073b9540618e4 [ 541.160016] env[64020]: DEBUG nova.compute.resource_tracker [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 541.160364] env[64020]: DEBUG nova.compute.resource_tracker [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 541.308345] env[64020]: INFO nova.scheduler.client.report [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] [req-336ee078-b846-4e23-ae28-6b4466955b9f] Created resource provider record via placement API for resource provider with UUID 092888c9-2221-4dfc-9104-eeeb335c764f and name domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28. [ 541.325026] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a933d756-b851-4bad-b76e-eedb40bc2821 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.332528] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b5a7c88-3f3c-4325-8c52-3bc06b4704d4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.360241] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df08a7f6-9e9c-476b-86c3-81ce6ada2ba6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.366988] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8f4da08-370a-4052-9730-eeec8371fffb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.380223] env[64020]: DEBUG nova.compute.provider_tree [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Updating inventory in ProviderTree for provider 092888c9-2221-4dfc-9104-eeeb335c764f with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 541.380799] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg 8700e03e74c140a581473d8cfb6e47e3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 541.387620] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8700e03e74c140a581473d8cfb6e47e3 [ 541.918408] env[64020]: DEBUG nova.scheduler.client.report [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Updated inventory for provider 092888c9-2221-4dfc-9104-eeeb335c764f with generation 0 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:971}} [ 541.918640] env[64020]: DEBUG nova.compute.provider_tree [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Updating resource provider 092888c9-2221-4dfc-9104-eeeb335c764f generation from 0 to 1 during operation: update_inventory {{(pid=64020) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 541.918778] env[64020]: DEBUG nova.compute.provider_tree [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Updating inventory in ProviderTree for provider 092888c9-2221-4dfc-9104-eeeb335c764f with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 541.970533] env[64020]: DEBUG nova.compute.provider_tree [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Updating resource provider 092888c9-2221-4dfc-9104-eeeb335c764f generation from 1 to 2 during operation: update_traits {{(pid=64020) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 541.972821] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Expecting reply to msg aa96c0721daf4b0caacfcc71a094da73 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 541.987348] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa96c0721daf4b0caacfcc71a094da73 [ 542.475494] env[64020]: DEBUG nova.compute.resource_tracker [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=64020) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 542.475851] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 3.831s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 542.475851] env[64020]: DEBUG nova.service [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Creating RPC server for service compute {{(pid=64020) start /opt/stack/nova/nova/service.py:186}} [ 542.485941] env[64020]: INFO oslo.messaging._drivers.impl_rabbit [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Creating fanout queue: compute_fanout_5b7308b8d75d48cea6baf3cf22ee1158 [ 542.488875] env[64020]: DEBUG nova.service [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] Join ServiceGroup membership for this service compute {{(pid=64020) start /opt/stack/nova/nova/service.py:203}} [ 542.489092] env[64020]: DEBUG nova.servicegroup.drivers.db [None req-dec5b246-d30f-42c7-b39f-600a3a8b7489 None None] DB_Driver: join new ServiceGroup member cpu-1 to the compute group, service = {{(pid=64020) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} [ 547.490669] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c6036395b10443c79b4610dfbfdc2ef5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 547.491784] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._sync_power_states {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 547.492400] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg cb6d81c1bc584b8ca9c8018330e05c0e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 547.502177] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c6036395b10443c79b4610dfbfdc2ef5 [ 547.505328] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cb6d81c1bc584b8ca9c8018330e05c0e [ 547.998545] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Getting list of instances from cluster (obj){ [ 547.998545] env[64020]: value = "domain-c8" [ 547.998545] env[64020]: _type = "ClusterComputeResource" [ 547.998545] env[64020]: } {{(pid=64020) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 547.999715] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2064daf-6e3a-48b2-82d1-5b1ddf6017a2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.008375] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Got total of 0 instances {{(pid=64020) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 548.008606] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._cleanup_running_deleted_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 548.008914] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Getting list of instances from cluster (obj){ [ 548.008914] env[64020]: value = "domain-c8" [ 548.008914] env[64020]: _type = "ClusterComputeResource" [ 548.008914] env[64020]: } {{(pid=64020) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 548.009806] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13e82244-b20c-431f-88be-7ca4d7cb10f2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.017056] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Got total of 0 instances {{(pid=64020) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 579.606949] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Acquiring lock "9d4a7822-643e-4bdb-b431-a16e5d183100" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 579.607238] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Lock "9d4a7822-643e-4bdb-b431-a16e5d183100" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 579.607672] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 8b4370ed9614485d93fa1bf9f94ffca4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 579.643879] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b4370ed9614485d93fa1bf9f94ffca4 [ 580.112172] env[64020]: DEBUG nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 580.113959] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 3768ee80ba1449148ef8692853eb738f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 580.200245] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3768ee80ba1449148ef8692853eb738f [ 580.648316] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 580.648636] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 580.650495] env[64020]: INFO nova.compute.claims [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 580.652381] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg c70f506f48cd4d30850da80478037764 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 580.721779] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c70f506f48cd4d30850da80478037764 [ 581.157292] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg f9c1f7d3523244e8a1f528b89cc6530a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 581.176702] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f9c1f7d3523244e8a1f528b89cc6530a [ 581.506413] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Acquiring lock "bbd58f57-9900-4d5d-847e-5275705e97f2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 581.506607] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Lock "bbd58f57-9900-4d5d-847e-5275705e97f2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 581.507099] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg 658ab6abb5ea47559adc75f30578b4cf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 581.523840] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 658ab6abb5ea47559adc75f30578b4cf [ 581.710760] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce9573a2-66e3-418c-b114-f0eb40f4ecd1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 581.720309] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04cebd41-4884-4093-adb4-ad8e593f02e0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 581.756770] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b51fd347-538e-4fd2-a208-3fb54741e857 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 581.776166] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42023b47-f0b0-40b4-872a-ccf3ef0875ab {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 581.792643] env[64020]: DEBUG nova.compute.provider_tree [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 581.793239] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 321f731eb9614dfb929a75130f0dd7f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 581.812289] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 321f731eb9614dfb929a75130f0dd7f5 [ 582.012559] env[64020]: DEBUG nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 582.012559] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg d58db2367d1e41fda0b33e3f213d3146 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 582.092594] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d58db2367d1e41fda0b33e3f213d3146 [ 582.296539] env[64020]: DEBUG nova.scheduler.client.report [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 582.298865] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 501385fa7b3e4cc59822c7008a52d779 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 582.350752] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 501385fa7b3e4cc59822c7008a52d779 [ 582.472703] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Acquiring lock "c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 582.472783] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Lock "c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 582.473242] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg a42ff663707c40129a6b3b7f413fe849 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 582.491253] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a42ff663707c40129a6b3b7f413fe849 [ 582.534406] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 582.552157] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquiring lock "4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 582.552566] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 582.552909] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 17ca5233263a46759ab88c8aa69b7195 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 582.564512] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 17ca5233263a46759ab88c8aa69b7195 [ 582.801917] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.153s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 582.802482] env[64020]: DEBUG nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 582.804196] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 6d7ce6ae2b344e30acc8e951bd195412 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 582.805180] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.271s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 582.806658] env[64020]: INFO nova.compute.claims [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 582.808315] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg 8f3f045b19af462b99d01567718b54b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 582.907418] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d7ce6ae2b344e30acc8e951bd195412 [ 582.913263] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f3f045b19af462b99d01567718b54b3 [ 582.975151] env[64020]: DEBUG nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 582.977161] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg eabbc58828f54c0490d52e2b6a148459 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 583.055870] env[64020]: DEBUG nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 583.057588] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 2c580ead972f42879748f663ed99f215 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 583.109396] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eabbc58828f54c0490d52e2b6a148459 [ 583.283212] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c580ead972f42879748f663ed99f215 [ 583.311598] env[64020]: DEBUG nova.compute.utils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 583.314438] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 912dd335a5d545518dfe67c0c4cc200f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 583.314438] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg fc1a03aa269e408384f840dcff55cfc9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 583.315560] env[64020]: DEBUG nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 583.315560] env[64020]: DEBUG nova.network.neutron [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 583.323562] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc1a03aa269e408384f840dcff55cfc9 [ 583.336291] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 912dd335a5d545518dfe67c0c4cc200f [ 583.515281] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 583.584195] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 583.816831] env[64020]: DEBUG nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 583.818751] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg bb6403f021ca43b2b47ec915d13aa5f0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 583.863706] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb6403f021ca43b2b47ec915d13aa5f0 [ 583.899277] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20807601-c6b9-48aa-ae70-edf7561a89ab {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 583.907835] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b662ed8-9d5d-45b4-9937-0795b97c7eb2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 583.957578] env[64020]: DEBUG nova.policy [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b5a48e010eb048bb82bd17e3682d3a25', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '46cd4934f3324c299ca14295c9777ccb', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 583.959831] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ca3b545-5667-41d2-88e5-c05732ef97b0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 583.969199] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e5599072-6d4e-4a21-a203-6e0ab67217aa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 583.983933] env[64020]: DEBUG nova.compute.provider_tree [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 583.984545] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg 54c7da05f2ec4b1586a4e99c96f65c6e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 583.993088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54c7da05f2ec4b1586a4e99c96f65c6e [ 584.324583] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 8d7e9d58979c4011a72dd2c197a00d5e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 584.387867] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d7e9d58979c4011a72dd2c197a00d5e [ 584.488393] env[64020]: DEBUG nova.scheduler.client.report [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 584.490703] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg 88bd591f76ad44419463ccd032590a3e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 584.506912] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88bd591f76ad44419463ccd032590a3e [ 584.754636] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 584.754876] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 584.755382] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg b1178d709e6c4fa48953868473124856 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 584.766638] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1178d709e6c4fa48953868473124856 [ 584.828817] env[64020]: DEBUG nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 584.861098] env[64020]: DEBUG nova.virt.hardware [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 584.861247] env[64020]: DEBUG nova.virt.hardware [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 584.861406] env[64020]: DEBUG nova.virt.hardware [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 584.861591] env[64020]: DEBUG nova.virt.hardware [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 584.861734] env[64020]: DEBUG nova.virt.hardware [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 584.861880] env[64020]: DEBUG nova.virt.hardware [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 584.862091] env[64020]: DEBUG nova.virt.hardware [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 584.862273] env[64020]: DEBUG nova.virt.hardware [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 584.862678] env[64020]: DEBUG nova.virt.hardware [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 584.862849] env[64020]: DEBUG nova.virt.hardware [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 584.863019] env[64020]: DEBUG nova.virt.hardware [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 584.863883] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2bce8668-d15b-4e2f-b5c3-eb5065c51567 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.872471] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aead1b6c-90de-4c23-807e-3ff12c00501f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.888969] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11dec87c-1237-4410-b27c-37b15c4fd86d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.992993] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.188s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 584.993573] env[64020]: DEBUG nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 584.996338] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg 38457a3b232d42019092f01997c6dd56 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 584.996603] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 1.481s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 584.997840] env[64020]: INFO nova.compute.claims [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 584.999668] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg 74bdfd73882042fc9abb8dd7690a5c66 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 585.054735] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 38457a3b232d42019092f01997c6dd56 [ 585.066179] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74bdfd73882042fc9abb8dd7690a5c66 [ 585.091861] env[64020]: DEBUG nova.network.neutron [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Successfully created port: e25137f1-b446-40b1-9e4b-0854030bbd51 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 585.264269] env[64020]: DEBUG nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 585.264269] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg cfab5e670d534ee8ae9a2d594a811753 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 585.315011] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfab5e670d534ee8ae9a2d594a811753 [ 585.502955] env[64020]: DEBUG nova.compute.utils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 585.503668] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg bc45d3b7ff6541549a7b45f84e907a77 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 585.504546] env[64020]: DEBUG nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 585.504752] env[64020]: DEBUG nova.network.neutron [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 585.513014] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg a453fb628cb946fb95c4a2d0d92c33a9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 585.518511] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc45d3b7ff6541549a7b45f84e907a77 [ 585.523482] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a453fb628cb946fb95c4a2d0d92c33a9 [ 585.678211] env[64020]: DEBUG nova.policy [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8da1307c17484a6d8ae75fc934ae5dbb', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8d3802d2d634409aae1e4e63785dacbc', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 585.743860] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Acquiring lock "b18fca61-2b17-4758-824d-e02d2cb342d8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.744078] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Lock "b18fca61-2b17-4758-824d-e02d2cb342d8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.744645] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 60bbad3dd9ab4c66a2f2bcf0344da938 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 585.759646] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60bbad3dd9ab4c66a2f2bcf0344da938 [ 585.785759] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 586.011877] env[64020]: DEBUG nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 586.011877] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg e312cdf0d01546f1a5710d43a2af5386 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 586.069028] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e312cdf0d01546f1a5710d43a2af5386 [ 586.120190] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3794911b-5e22-4ab1-9d36-9d2b60b9af0b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.128613] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3dc4bed1-d579-4907-afad-10d448fedbc1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.165581] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6283b97c-7dff-426e-b1f3-bd044b98b865 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.174117] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0e52548-a90f-477b-b0e5-e43969791bc1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.189662] env[64020]: DEBUG nova.compute.provider_tree [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 586.190512] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg ce4a7d993b6a4bde98ccefb25e8a70dd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 586.198482] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce4a7d993b6a4bde98ccefb25e8a70dd [ 586.246703] env[64020]: DEBUG nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 586.248541] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 845444e5d6d2430bb0212e288a4de7a3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 586.303272] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 845444e5d6d2430bb0212e288a4de7a3 [ 586.517161] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg c06c9b80feae4b2892631a5366dfe419 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 586.573587] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c06c9b80feae4b2892631a5366dfe419 [ 586.693728] env[64020]: DEBUG nova.scheduler.client.report [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 586.696317] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg bde7d182dd924110aab9f55bb35cd35c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 586.708559] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bde7d182dd924110aab9f55bb35cd35c [ 586.766704] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 586.952587] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Acquiring lock "7529a478-7efe-4dff-8429-810549f56bd7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 586.952587] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Lock "7529a478-7efe-4dff-8429-810549f56bd7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 586.952957] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg e7efb09e43c94150b2c042b8f15bc12c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 586.970398] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e7efb09e43c94150b2c042b8f15bc12c [ 587.022445] env[64020]: DEBUG nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 587.061784] env[64020]: DEBUG nova.virt.hardware [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 587.062010] env[64020]: DEBUG nova.virt.hardware [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 587.062189] env[64020]: DEBUG nova.virt.hardware [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 587.062389] env[64020]: DEBUG nova.virt.hardware [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 587.062533] env[64020]: DEBUG nova.virt.hardware [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 587.062669] env[64020]: DEBUG nova.virt.hardware [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 587.062868] env[64020]: DEBUG nova.virt.hardware [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 587.063053] env[64020]: DEBUG nova.virt.hardware [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 587.063194] env[64020]: DEBUG nova.virt.hardware [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 587.063329] env[64020]: DEBUG nova.virt.hardware [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 587.063488] env[64020]: DEBUG nova.virt.hardware [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 587.064405] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-762f8302-0e07-4c5e-9554-3c7e898418f0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.075631] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27bbc169-3015-4a16-8e27-437007dff7f0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.199273] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.203s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 587.199846] env[64020]: DEBUG nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 587.201759] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg 471bdd248dd84fa48e92ec33fd37eb7a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 587.202926] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.619s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 587.204557] env[64020]: INFO nova.compute.claims [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 587.206048] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 8d12c0bf887843a8bf9e22969c104b75 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 587.258966] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d12c0bf887843a8bf9e22969c104b75 [ 587.260107] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 471bdd248dd84fa48e92ec33fd37eb7a [ 587.291751] env[64020]: DEBUG nova.network.neutron [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Successfully created port: 32bd22fb-e5de-4222-8ff3-1328a2144ded {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 587.455637] env[64020]: DEBUG nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 587.457443] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 8ba8b82fe5a74e69aee6fc364b9fe60c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 587.496079] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ba8b82fe5a74e69aee6fc364b9fe60c [ 587.694440] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquiring lock "5290c09b-d77c-4d27-b367-9de52f54d16a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 587.694878] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "5290c09b-d77c-4d27-b367-9de52f54d16a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 587.695726] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg dc1433673a2743959e75b2024170fb0f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 587.705661] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc1433673a2743959e75b2024170fb0f [ 587.708027] env[64020]: DEBUG nova.compute.utils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 587.708634] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg 93b95ad0ec9b414bac2871bb56b02634 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 587.709550] env[64020]: DEBUG nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 587.709731] env[64020]: DEBUG nova.network.neutron [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 587.713079] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 7ce93e34c7434e01a83ceb56a7df090f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 587.724258] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ce93e34c7434e01a83ceb56a7df090f [ 587.724829] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93b95ad0ec9b414bac2871bb56b02634 [ 587.856730] env[64020]: DEBUG nova.policy [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9d46ccb900ce42c3967dc85277ddd348', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0aba900f457941238fec4fe58144f89f', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 587.986672] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 588.197995] env[64020]: DEBUG nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 588.199745] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg a9f7fce650bf47f188c5499728afd43c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 588.215226] env[64020]: DEBUG nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 588.217116] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg 592399bf85374162b5d9c32a2c5f27f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 588.237862] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a9f7fce650bf47f188c5499728afd43c [ 588.267368] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 592399bf85374162b5d9c32a2c5f27f5 [ 588.350205] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d268c64-98bc-45c3-9ab3-dfe57f20dc17 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.359627] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f520a96a-f1ec-40dd-b7bb-d20426886527 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.397879] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48656140-bee9-4c77-9e5b-f4ec361d1d32 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.406157] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae334632-50f3-4d0c-9924-d268e703cbdf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.421520] env[64020]: DEBUG nova.compute.provider_tree [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 588.422051] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg bea40654c5574305b407be2831e3a38a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 588.436802] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bea40654c5574305b407be2831e3a38a [ 588.718791] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 588.724991] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg c89662fa09c348ccad2dd5ed7be75007 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 588.768458] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c89662fa09c348ccad2dd5ed7be75007 [ 588.925232] env[64020]: DEBUG nova.scheduler.client.report [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 588.928203] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 2185d53f9a6b4396b8a0ae6f22dddbc3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 588.947474] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2185d53f9a6b4396b8a0ae6f22dddbc3 [ 588.989396] env[64020]: DEBUG nova.network.neutron [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Successfully created port: 06ebe7b0-7b82-4f26-85b9-cc69537dc66f {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 589.228827] env[64020]: DEBUG nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 589.258811] env[64020]: DEBUG nova.virt.hardware [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 589.259171] env[64020]: DEBUG nova.virt.hardware [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 589.259451] env[64020]: DEBUG nova.virt.hardware [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 589.259644] env[64020]: DEBUG nova.virt.hardware [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 589.259784] env[64020]: DEBUG nova.virt.hardware [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 589.259925] env[64020]: DEBUG nova.virt.hardware [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 589.260749] env[64020]: DEBUG nova.virt.hardware [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 589.260979] env[64020]: DEBUG nova.virt.hardware [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 589.261315] env[64020]: DEBUG nova.virt.hardware [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 589.262331] env[64020]: DEBUG nova.virt.hardware [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 589.262331] env[64020]: DEBUG nova.virt.hardware [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 589.262902] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87788877-555a-4168-afbc-96ad76379e5c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.275054] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13993800-6fac-47c5-9d1f-43d7184faa94 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.444767] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.232s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 589.444767] env[64020]: DEBUG nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 589.444767] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 53e89103b90b4015ab4ea5c8a022441d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 589.444767] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.653s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.444767] env[64020]: INFO nova.compute.claims [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 589.445048] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 7fb3ad315967479eac830ad21988af76 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 589.485890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 53e89103b90b4015ab4ea5c8a022441d [ 589.494856] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7fb3ad315967479eac830ad21988af76 [ 589.931798] env[64020]: ERROR nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e25137f1-b446-40b1-9e4b-0854030bbd51, please check neutron logs for more information. [ 589.931798] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 589.931798] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 589.931798] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 589.931798] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 589.931798] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 589.931798] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 589.931798] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 589.931798] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 589.931798] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 589.931798] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 589.931798] env[64020]: ERROR nova.compute.manager raise self.value [ 589.931798] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 589.931798] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 589.931798] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 589.931798] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 589.932240] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 589.932240] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 589.932240] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e25137f1-b446-40b1-9e4b-0854030bbd51, please check neutron logs for more information. [ 589.932240] env[64020]: ERROR nova.compute.manager [ 589.932240] env[64020]: Traceback (most recent call last): [ 589.932240] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 589.932240] env[64020]: listener.cb(fileno) [ 589.932240] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 589.932240] env[64020]: result = function(*args, **kwargs) [ 589.932240] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 589.932240] env[64020]: return func(*args, **kwargs) [ 589.932240] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 589.932240] env[64020]: raise e [ 589.932240] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 589.932240] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 589.932240] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 589.932240] env[64020]: created_port_ids = self._update_ports_for_instance( [ 589.932240] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 589.932240] env[64020]: with excutils.save_and_reraise_exception(): [ 589.932240] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 589.932240] env[64020]: self.force_reraise() [ 589.932240] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 589.932240] env[64020]: raise self.value [ 589.932240] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 589.932240] env[64020]: updated_port = self._update_port( [ 589.932240] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 589.932240] env[64020]: _ensure_no_port_binding_failure(port) [ 589.932240] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 589.932240] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 589.932978] env[64020]: nova.exception.PortBindingFailed: Binding failed for port e25137f1-b446-40b1-9e4b-0854030bbd51, please check neutron logs for more information. [ 589.932978] env[64020]: Removing descriptor: 15 [ 589.934054] env[64020]: ERROR nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e25137f1-b446-40b1-9e4b-0854030bbd51, please check neutron logs for more information. [ 589.934054] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Traceback (most recent call last): [ 589.934054] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 589.934054] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] yield resources [ 589.934054] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 589.934054] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] self.driver.spawn(context, instance, image_meta, [ 589.934054] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 589.934054] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] self._vmops.spawn(context, instance, image_meta, injected_files, [ 589.934054] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 589.934054] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] vm_ref = self.build_virtual_machine(instance, [ 589.934054] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] vif_infos = vmwarevif.get_vif_info(self._session, [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] for vif in network_info: [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] return self._sync_wrapper(fn, *args, **kwargs) [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] self.wait() [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] self[:] = self._gt.wait() [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] return self._exit_event.wait() [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 589.934380] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] result = hub.switch() [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] return self.greenlet.switch() [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] result = function(*args, **kwargs) [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] return func(*args, **kwargs) [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] raise e [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] nwinfo = self.network_api.allocate_for_instance( [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] created_port_ids = self._update_ports_for_instance( [ 589.934708] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] with excutils.save_and_reraise_exception(): [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] self.force_reraise() [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] raise self.value [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] updated_port = self._update_port( [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] _ensure_no_port_binding_failure(port) [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] raise exception.PortBindingFailed(port_id=port['id']) [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] nova.exception.PortBindingFailed: Binding failed for port e25137f1-b446-40b1-9e4b-0854030bbd51, please check neutron logs for more information. [ 589.935021] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] [ 589.935334] env[64020]: INFO nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Terminating instance [ 589.937014] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Acquiring lock "refresh_cache-9d4a7822-643e-4bdb-b431-a16e5d183100" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 589.937289] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Acquired lock "refresh_cache-9d4a7822-643e-4bdb-b431-a16e5d183100" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 589.937580] env[64020]: DEBUG nova.network.neutron [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 589.938062] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg ff107f27e8c140ebb481d2332049a2e1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 589.946444] env[64020]: DEBUG nova.compute.utils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 589.947031] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 1d1b0234427047308c894f03c3335797 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 589.948030] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff107f27e8c140ebb481d2332049a2e1 [ 589.949528] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 32d017df7d0b4e6e94d75a4739602dd9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 589.951218] env[64020]: DEBUG nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 589.951383] env[64020]: DEBUG nova.network.neutron [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 589.996148] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d1b0234427047308c894f03c3335797 [ 589.996705] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 32d017df7d0b4e6e94d75a4739602dd9 [ 590.110911] env[64020]: DEBUG nova.policy [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fc309fb9d0604703a2c159d2f52affe3', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e9f7ec1e87ee49bc9c39245e2d99c402', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 590.457126] env[64020]: DEBUG nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 590.458961] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 74c15eef4fd44544aff94a2171a12b70 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 590.499757] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74c15eef4fd44544aff94a2171a12b70 [ 590.506009] env[64020]: DEBUG nova.network.neutron [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 590.581545] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51ecdd17-0cf8-4d50-8764-f4549e21021d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.590009] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e7ae841-5204-4ff7-8f4a-9604a0e982e5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.630178] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-23ff2d9d-bbd6-4095-b28a-ead3fb623d1a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.643243] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cc83899-7621-4777-97c7-8a9fdfc52cd8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.657425] env[64020]: DEBUG nova.compute.provider_tree [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 590.658035] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg b2ccb783caf740abaa9c97e620f0ab7f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 590.666961] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b2ccb783caf740abaa9c97e620f0ab7f [ 590.684936] env[64020]: ERROR nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 32bd22fb-e5de-4222-8ff3-1328a2144ded, please check neutron logs for more information. [ 590.684936] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 590.684936] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 590.684936] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 590.684936] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 590.684936] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 590.684936] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 590.684936] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 590.684936] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 590.684936] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 590.684936] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 590.684936] env[64020]: ERROR nova.compute.manager raise self.value [ 590.684936] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 590.684936] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 590.684936] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 590.684936] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 590.685393] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 590.685393] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 590.685393] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 32bd22fb-e5de-4222-8ff3-1328a2144ded, please check neutron logs for more information. [ 590.685393] env[64020]: ERROR nova.compute.manager [ 590.685393] env[64020]: Traceback (most recent call last): [ 590.685393] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 590.685393] env[64020]: listener.cb(fileno) [ 590.685393] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 590.685393] env[64020]: result = function(*args, **kwargs) [ 590.685393] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 590.685393] env[64020]: return func(*args, **kwargs) [ 590.685393] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 590.685393] env[64020]: raise e [ 590.685393] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 590.685393] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 590.685393] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 590.685393] env[64020]: created_port_ids = self._update_ports_for_instance( [ 590.685393] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 590.685393] env[64020]: with excutils.save_and_reraise_exception(): [ 590.685393] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 590.685393] env[64020]: self.force_reraise() [ 590.685393] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 590.685393] env[64020]: raise self.value [ 590.685393] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 590.685393] env[64020]: updated_port = self._update_port( [ 590.685393] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 590.685393] env[64020]: _ensure_no_port_binding_failure(port) [ 590.685393] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 590.685393] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 590.686204] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 32bd22fb-e5de-4222-8ff3-1328a2144ded, please check neutron logs for more information. [ 590.686204] env[64020]: Removing descriptor: 16 [ 590.686437] env[64020]: ERROR nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 32bd22fb-e5de-4222-8ff3-1328a2144ded, please check neutron logs for more information. [ 590.686437] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Traceback (most recent call last): [ 590.686437] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 590.686437] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] yield resources [ 590.686437] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 590.686437] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] self.driver.spawn(context, instance, image_meta, [ 590.686437] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 590.686437] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 590.686437] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 590.686437] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] vm_ref = self.build_virtual_machine(instance, [ 590.686437] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] vif_infos = vmwarevif.get_vif_info(self._session, [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] for vif in network_info: [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] return self._sync_wrapper(fn, *args, **kwargs) [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] self.wait() [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] self[:] = self._gt.wait() [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] return self._exit_event.wait() [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 590.686801] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] result = hub.switch() [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] return self.greenlet.switch() [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] result = function(*args, **kwargs) [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] return func(*args, **kwargs) [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] raise e [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] nwinfo = self.network_api.allocate_for_instance( [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] created_port_ids = self._update_ports_for_instance( [ 590.687153] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] with excutils.save_and_reraise_exception(): [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] self.force_reraise() [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] raise self.value [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] updated_port = self._update_port( [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] _ensure_no_port_binding_failure(port) [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] raise exception.PortBindingFailed(port_id=port['id']) [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] nova.exception.PortBindingFailed: Binding failed for port 32bd22fb-e5de-4222-8ff3-1328a2144ded, please check neutron logs for more information. [ 590.687586] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] [ 590.687968] env[64020]: INFO nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Terminating instance [ 590.690329] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Acquiring lock "refresh_cache-bbd58f57-9900-4d5d-847e-5275705e97f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 590.690675] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Acquired lock "refresh_cache-bbd58f57-9900-4d5d-847e-5275705e97f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 590.690896] env[64020]: DEBUG nova.network.neutron [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 590.691387] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg b45f0620e88c4ca2986eb86fbb2c25c2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 590.700847] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b45f0620e88c4ca2986eb86fbb2c25c2 [ 590.802892] env[64020]: DEBUG nova.network.neutron [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 590.802892] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg cb77d13eb0f64645b3db9cf860f5df85 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 590.815304] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cb77d13eb0f64645b3db9cf860f5df85 [ 590.878471] env[64020]: DEBUG nova.compute.manager [req-449f1b51-deb9-43f4-8280-27ee37905292 req-b3d85ce5-1f2d-4959-a2d4-cf545547301a service nova] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Received event network-changed-e25137f1-b446-40b1-9e4b-0854030bbd51 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 590.878471] env[64020]: DEBUG nova.compute.manager [req-449f1b51-deb9-43f4-8280-27ee37905292 req-b3d85ce5-1f2d-4959-a2d4-cf545547301a service nova] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Refreshing instance network info cache due to event network-changed-e25137f1-b446-40b1-9e4b-0854030bbd51. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 590.878471] env[64020]: DEBUG oslo_concurrency.lockutils [req-449f1b51-deb9-43f4-8280-27ee37905292 req-b3d85ce5-1f2d-4959-a2d4-cf545547301a service nova] Acquiring lock "refresh_cache-9d4a7822-643e-4bdb-b431-a16e5d183100" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 590.973400] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 78efbe60eec94522b7e6162e148bbe70 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 591.015213] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 78efbe60eec94522b7e6162e148bbe70 [ 591.105846] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.106161] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.106352] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Starting heal instance info cache {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 591.106473] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Rebuilding the list of instances to heal {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 591.107096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 1e051c6145654ddc8f269eeb4432626f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 591.127066] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e051c6145654ddc8f269eeb4432626f [ 591.134398] env[64020]: DEBUG nova.network.neutron [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Successfully created port: 4c2e20e6-3e5d-4866-b20c-469a896012eb {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 591.161017] env[64020]: DEBUG nova.scheduler.client.report [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 591.163474] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 7b3d8912260d421caf491e50b4ffd1b6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 591.178015] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7b3d8912260d421caf491e50b4ffd1b6 [ 591.279915] env[64020]: DEBUG nova.network.neutron [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 591.304532] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Releasing lock "refresh_cache-9d4a7822-643e-4bdb-b431-a16e5d183100" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 591.304963] env[64020]: DEBUG nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 591.305149] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 591.305457] env[64020]: DEBUG oslo_concurrency.lockutils [req-449f1b51-deb9-43f4-8280-27ee37905292 req-b3d85ce5-1f2d-4959-a2d4-cf545547301a service nova] Acquired lock "refresh_cache-9d4a7822-643e-4bdb-b431-a16e5d183100" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 591.305618] env[64020]: DEBUG nova.network.neutron [req-449f1b51-deb9-43f4-8280-27ee37905292 req-b3d85ce5-1f2d-4959-a2d4-cf545547301a service nova] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Refreshing network info cache for port e25137f1-b446-40b1-9e4b-0854030bbd51 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 591.306044] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-449f1b51-deb9-43f4-8280-27ee37905292 req-b3d85ce5-1f2d-4959-a2d4-cf545547301a service nova] Expecting reply to msg d0326a7959fa4e5bb72f4b2bee5aa824 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 591.306843] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-dabdadbb-991f-46a3-a8f6-832c112033af {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.316715] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1a2df7c-5fbb-49c3-bb3b-45c5b272939b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.331351] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0326a7959fa4e5bb72f4b2bee5aa824 [ 591.345357] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9d4a7822-643e-4bdb-b431-a16e5d183100 could not be found. [ 591.345657] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 591.346099] env[64020]: INFO nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Took 0.04 seconds to destroy the instance on the hypervisor. [ 591.347437] env[64020]: DEBUG oslo.service.loopingcall [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 591.347437] env[64020]: DEBUG nova.compute.manager [-] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 591.347437] env[64020]: DEBUG nova.network.neutron [-] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 591.371553] env[64020]: DEBUG nova.network.neutron [-] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 591.371553] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8ffadf0e7b3f45aba86ab2521ffd34e5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 591.377298] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ffadf0e7b3f45aba86ab2521ffd34e5 [ 591.475495] env[64020]: DEBUG nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 591.504455] env[64020]: DEBUG nova.virt.hardware [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 591.504455] env[64020]: DEBUG nova.virt.hardware [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 591.504697] env[64020]: DEBUG nova.virt.hardware [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 591.504773] env[64020]: DEBUG nova.virt.hardware [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 591.504896] env[64020]: DEBUG nova.virt.hardware [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 591.505042] env[64020]: DEBUG nova.virt.hardware [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 591.505244] env[64020]: DEBUG nova.virt.hardware [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 591.505400] env[64020]: DEBUG nova.virt.hardware [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 591.505559] env[64020]: DEBUG nova.virt.hardware [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 591.505715] env[64020]: DEBUG nova.virt.hardware [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 591.505880] env[64020]: DEBUG nova.virt.hardware [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 591.506729] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d7d8836-92b0-422c-8620-e131581ee383 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.516945] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e312ebff-e4d7-4f29-a927-1ea006fcbae5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.610437] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 591.610608] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 591.610739] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 591.610862] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 591.610980] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 591.611168] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Didn't find any instances for network info cache update. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 591.611379] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.611673] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.611990] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.611990] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.612245] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.612443] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.612628] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=64020) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 591.612748] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager.update_available_resource {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.613097] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 0e5ab1129785450fb38f18f1c9e9c2c0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 591.624215] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e5ab1129785450fb38f18f1c9e9c2c0 [ 591.638661] env[64020]: DEBUG nova.compute.manager [req-db547082-4b53-4bb6-a4a6-ea294683b8f8 req-afe726a1-df5e-4f7c-a660-6e8514d563f5 service nova] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Received event network-changed-32bd22fb-e5de-4222-8ff3-1328a2144ded {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 591.639250] env[64020]: DEBUG nova.compute.manager [req-db547082-4b53-4bb6-a4a6-ea294683b8f8 req-afe726a1-df5e-4f7c-a660-6e8514d563f5 service nova] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Refreshing instance network info cache due to event network-changed-32bd22fb-e5de-4222-8ff3-1328a2144ded. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 591.639456] env[64020]: DEBUG oslo_concurrency.lockutils [req-db547082-4b53-4bb6-a4a6-ea294683b8f8 req-afe726a1-df5e-4f7c-a660-6e8514d563f5 service nova] Acquiring lock "refresh_cache-bbd58f57-9900-4d5d-847e-5275705e97f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 591.665791] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.227s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 591.666297] env[64020]: DEBUG nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 591.667978] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg be267110eac1448dbff8d245201bce3d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 591.668951] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 4.902s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 591.671749] env[64020]: INFO nova.compute.claims [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 591.673523] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg c093ca6d92b64cdaae31c64d9915d02e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 591.716867] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be267110eac1448dbff8d245201bce3d [ 591.718447] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c093ca6d92b64cdaae31c64d9915d02e [ 591.871870] env[64020]: DEBUG nova.network.neutron [req-449f1b51-deb9-43f4-8280-27ee37905292 req-b3d85ce5-1f2d-4959-a2d4-cf545547301a service nova] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 591.875344] env[64020]: DEBUG nova.network.neutron [-] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 591.876165] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6d061421751148cab375192f7ef84baf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 591.886220] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d061421751148cab375192f7ef84baf [ 591.909001] env[64020]: DEBUG nova.network.neutron [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 591.909455] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg a425a86b128f4bbdbff667166e243bb4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 591.923946] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a425a86b128f4bbdbff667166e243bb4 [ 591.999362] env[64020]: ERROR nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 06ebe7b0-7b82-4f26-85b9-cc69537dc66f, please check neutron logs for more information. [ 591.999362] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 591.999362] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 591.999362] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 591.999362] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 591.999362] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 591.999362] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 591.999362] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 591.999362] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 591.999362] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 591.999362] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 591.999362] env[64020]: ERROR nova.compute.manager raise self.value [ 591.999362] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 591.999362] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 591.999362] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 591.999362] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 591.999893] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 591.999893] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 591.999893] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 06ebe7b0-7b82-4f26-85b9-cc69537dc66f, please check neutron logs for more information. [ 591.999893] env[64020]: ERROR nova.compute.manager [ 591.999893] env[64020]: Traceback (most recent call last): [ 591.999893] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 591.999893] env[64020]: listener.cb(fileno) [ 591.999893] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 591.999893] env[64020]: result = function(*args, **kwargs) [ 591.999893] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 591.999893] env[64020]: return func(*args, **kwargs) [ 591.999893] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 591.999893] env[64020]: raise e [ 591.999893] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 591.999893] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 591.999893] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 591.999893] env[64020]: created_port_ids = self._update_ports_for_instance( [ 591.999893] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 591.999893] env[64020]: with excutils.save_and_reraise_exception(): [ 591.999893] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 591.999893] env[64020]: self.force_reraise() [ 591.999893] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 591.999893] env[64020]: raise self.value [ 591.999893] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 591.999893] env[64020]: updated_port = self._update_port( [ 591.999893] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 591.999893] env[64020]: _ensure_no_port_binding_failure(port) [ 591.999893] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 591.999893] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 592.000705] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 06ebe7b0-7b82-4f26-85b9-cc69537dc66f, please check neutron logs for more information. [ 592.000705] env[64020]: Removing descriptor: 17 [ 592.000705] env[64020]: ERROR nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 06ebe7b0-7b82-4f26-85b9-cc69537dc66f, please check neutron logs for more information. [ 592.000705] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Traceback (most recent call last): [ 592.000705] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 592.000705] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] yield resources [ 592.000705] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 592.000705] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] self.driver.spawn(context, instance, image_meta, [ 592.000705] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 592.000705] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 592.000705] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 592.000705] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] vm_ref = self.build_virtual_machine(instance, [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] vif_infos = vmwarevif.get_vif_info(self._session, [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] for vif in network_info: [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] return self._sync_wrapper(fn, *args, **kwargs) [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] self.wait() [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] self[:] = self._gt.wait() [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] return self._exit_event.wait() [ 592.001085] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] result = hub.switch() [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] return self.greenlet.switch() [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] result = function(*args, **kwargs) [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] return func(*args, **kwargs) [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] raise e [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] nwinfo = self.network_api.allocate_for_instance( [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 592.001627] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] created_port_ids = self._update_ports_for_instance( [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] with excutils.save_and_reraise_exception(): [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] self.force_reraise() [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] raise self.value [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] updated_port = self._update_port( [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] _ensure_no_port_binding_failure(port) [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 592.001987] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] raise exception.PortBindingFailed(port_id=port['id']) [ 592.002673] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] nova.exception.PortBindingFailed: Binding failed for port 06ebe7b0-7b82-4f26-85b9-cc69537dc66f, please check neutron logs for more information. [ 592.002673] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] [ 592.002673] env[64020]: INFO nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Terminating instance [ 592.007470] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Acquiring lock "refresh_cache-c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 592.007470] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Acquired lock "refresh_cache-c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 592.007470] env[64020]: DEBUG nova.network.neutron [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 592.007703] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg a6c823055a614a97b8d6a92647bbc3ce in queue reply_57893177120949e6a93cb88e15cd42b4 [ 592.018201] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6c823055a614a97b8d6a92647bbc3ce [ 592.056508] env[64020]: DEBUG nova.network.neutron [req-449f1b51-deb9-43f4-8280-27ee37905292 req-b3d85ce5-1f2d-4959-a2d4-cf545547301a service nova] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 592.057233] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-449f1b51-deb9-43f4-8280-27ee37905292 req-b3d85ce5-1f2d-4959-a2d4-cf545547301a service nova] Expecting reply to msg 22652c711cce4d5ebeb065c93af28fcb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 592.067489] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 22652c711cce4d5ebeb065c93af28fcb [ 592.115734] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 592.179641] env[64020]: DEBUG nova.compute.utils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 592.179893] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg b6fd48ee93ec454caa15f76d14d9a54d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 592.181921] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 9de5e480494b46b08a5e4f1b73894ae5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 592.182698] env[64020]: DEBUG nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 592.182852] env[64020]: DEBUG nova.network.neutron [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 592.189927] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9de5e480494b46b08a5e4f1b73894ae5 [ 592.192356] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b6fd48ee93ec454caa15f76d14d9a54d [ 592.382510] env[64020]: INFO nova.compute.manager [-] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Took 1.04 seconds to deallocate network for instance. [ 592.385081] env[64020]: DEBUG nova.compute.claims [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 592.385258] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 592.413037] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Releasing lock "refresh_cache-bbd58f57-9900-4d5d-847e-5275705e97f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 592.413510] env[64020]: DEBUG nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 592.413688] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 592.414045] env[64020]: DEBUG oslo_concurrency.lockutils [req-db547082-4b53-4bb6-a4a6-ea294683b8f8 req-afe726a1-df5e-4f7c-a660-6e8514d563f5 service nova] Acquired lock "refresh_cache-bbd58f57-9900-4d5d-847e-5275705e97f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 592.414227] env[64020]: DEBUG nova.network.neutron [req-db547082-4b53-4bb6-a4a6-ea294683b8f8 req-afe726a1-df5e-4f7c-a660-6e8514d563f5 service nova] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Refreshing network info cache for port 32bd22fb-e5de-4222-8ff3-1328a2144ded {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 592.414668] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-db547082-4b53-4bb6-a4a6-ea294683b8f8 req-afe726a1-df5e-4f7c-a660-6e8514d563f5 service nova] Expecting reply to msg 500b35c624a24ba7803790206f9d60b5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 592.415694] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b19e554c-ae02-458f-ba36-0d49bd31b3f7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.424753] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38e220fe-cf62-4993-bccf-ba07a74425ad {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.436096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 500b35c624a24ba7803790206f9d60b5 [ 592.458811] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance bbd58f57-9900-4d5d-847e-5275705e97f2 could not be found. [ 592.459094] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 592.459278] env[64020]: INFO nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Took 0.05 seconds to destroy the instance on the hypervisor. [ 592.459520] env[64020]: DEBUG oslo.service.loopingcall [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 592.459735] env[64020]: DEBUG nova.compute.manager [-] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 592.459820] env[64020]: DEBUG nova.network.neutron [-] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 592.472603] env[64020]: DEBUG nova.policy [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '29faa85bd7904cd985fb32f53b533c5b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '525215939f6741cea749e7d9e53f379e', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 592.525135] env[64020]: DEBUG nova.network.neutron [-] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 592.525645] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg d94fadd176644afdbea0116e921eb3fa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 592.538422] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d94fadd176644afdbea0116e921eb3fa [ 592.550310] env[64020]: DEBUG nova.network.neutron [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 592.559407] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquiring lock "b3440b55-2469-46fa-ac2c-3e207bf530ec" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 592.559622] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "b3440b55-2469-46fa-ac2c-3e207bf530ec" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 592.560070] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 668566a0293b477a8b7132b403a30e72 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 592.560924] env[64020]: DEBUG oslo_concurrency.lockutils [req-449f1b51-deb9-43f4-8280-27ee37905292 req-b3d85ce5-1f2d-4959-a2d4-cf545547301a service nova] Releasing lock "refresh_cache-9d4a7822-643e-4bdb-b431-a16e5d183100" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 592.570987] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 668566a0293b477a8b7132b403a30e72 [ 592.690114] env[64020]: DEBUG nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 592.700817] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg c1a6bade66604c57aedce675158dabf3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 592.753546] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c1a6bade66604c57aedce675158dabf3 [ 592.860261] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a4fc268-ab52-4308-af9a-4f800aea9b59 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.870556] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5f56b84-6b5a-4c0b-9590-c9a9628d2ada {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.916160] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83a49e8a-9afb-4669-ba0f-cc3bed213651 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.924140] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af043a5e-da5a-4444-84d7-30022384a0c2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.940079] env[64020]: DEBUG nova.compute.provider_tree [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 592.940624] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg a751e3fe0c5441c2914007935217266c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 592.953834] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a751e3fe0c5441c2914007935217266c [ 592.986939] env[64020]: DEBUG nova.network.neutron [req-db547082-4b53-4bb6-a4a6-ea294683b8f8 req-afe726a1-df5e-4f7c-a660-6e8514d563f5 service nova] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 593.029296] env[64020]: DEBUG nova.network.neutron [-] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 593.029296] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 67d70305c7e344b7bec7d3c7492fe1ec in queue reply_57893177120949e6a93cb88e15cd42b4 [ 593.041400] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 67d70305c7e344b7bec7d3c7492fe1ec [ 593.064535] env[64020]: DEBUG nova.compute.manager [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 593.064535] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg d162b2b74fe24e9e9c32eb440f341ad8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 593.103119] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d162b2b74fe24e9e9c32eb440f341ad8 [ 593.181182] env[64020]: DEBUG nova.network.neutron [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 593.181709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg aa930e4d92b8444b80e7fe3971e649c6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 593.191888] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa930e4d92b8444b80e7fe3971e649c6 [ 593.196029] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg c1a7427450844006b9c4e30654f5a642 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 593.260769] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c1a7427450844006b9c4e30654f5a642 [ 593.304429] env[64020]: DEBUG nova.network.neutron [req-db547082-4b53-4bb6-a4a6-ea294683b8f8 req-afe726a1-df5e-4f7c-a660-6e8514d563f5 service nova] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 593.305009] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-db547082-4b53-4bb6-a4a6-ea294683b8f8 req-afe726a1-df5e-4f7c-a660-6e8514d563f5 service nova] Expecting reply to msg 6327507431d0468abef2cd7dfe0c6568 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 593.317529] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6327507431d0468abef2cd7dfe0c6568 [ 593.443470] env[64020]: DEBUG nova.scheduler.client.report [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 593.445795] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 57af8e828ce441ada31817dd45d0d603 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 593.461937] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 57af8e828ce441ada31817dd45d0d603 [ 593.465220] env[64020]: DEBUG nova.network.neutron [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Successfully created port: 3a25c429-30d6-43fb-927d-555318c582a6 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 593.531165] env[64020]: INFO nova.compute.manager [-] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Took 1.07 seconds to deallocate network for instance. [ 593.536230] env[64020]: DEBUG nova.compute.claims [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 593.536368] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 593.563589] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Acquiring lock "1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 593.563805] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Lock "1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 593.564639] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg d72b4c75443a4b9d983d58bc4761fa67 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 593.580327] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d72b4c75443a4b9d983d58bc4761fa67 [ 593.584626] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 593.687310] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Releasing lock "refresh_cache-c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 593.687310] env[64020]: DEBUG nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 593.687310] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 593.687310] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-45962a21-98ae-4962-ae66-8902cc78e710 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 593.699867] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e77101b-afdf-4e40-9382-99b04d9a2761 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 593.718295] env[64020]: DEBUG nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 593.734154] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c could not be found. [ 593.734412] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 593.734636] env[64020]: INFO nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Took 0.05 seconds to destroy the instance on the hypervisor. [ 593.734910] env[64020]: DEBUG oslo.service.loopingcall [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 593.736944] env[64020]: DEBUG nova.compute.manager [-] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 593.737080] env[64020]: DEBUG nova.network.neutron [-] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 593.744317] env[64020]: DEBUG nova.virt.hardware [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 593.744585] env[64020]: DEBUG nova.virt.hardware [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 593.744812] env[64020]: DEBUG nova.virt.hardware [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 593.745015] env[64020]: DEBUG nova.virt.hardware [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 593.745305] env[64020]: DEBUG nova.virt.hardware [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 593.745486] env[64020]: DEBUG nova.virt.hardware [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 593.745749] env[64020]: DEBUG nova.virt.hardware [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 593.745937] env[64020]: DEBUG nova.virt.hardware [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 593.746136] env[64020]: DEBUG nova.virt.hardware [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 593.746390] env[64020]: DEBUG nova.virt.hardware [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 593.746655] env[64020]: DEBUG nova.virt.hardware [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 593.747552] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c79355d2-8d75-49b1-af32-64232677fae4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 593.755883] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-863c8a21-b960-491b-8d48-7891a29982c9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 593.813174] env[64020]: DEBUG nova.network.neutron [-] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 593.813740] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 55366b884fea4206a105b5cbffadfe73 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 593.817363] env[64020]: DEBUG oslo_concurrency.lockutils [req-db547082-4b53-4bb6-a4a6-ea294683b8f8 req-afe726a1-df5e-4f7c-a660-6e8514d563f5 service nova] Releasing lock "refresh_cache-bbd58f57-9900-4d5d-847e-5275705e97f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 593.825267] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 55366b884fea4206a105b5cbffadfe73 [ 593.939117] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquiring lock "65be7d88-0c12-4ad3-adc7-d089a1b91c28" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 593.939544] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "65be7d88-0c12-4ad3-adc7-d089a1b91c28" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 593.948344] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.279s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 593.949051] env[64020]: DEBUG nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 593.951212] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 6c096701287d48278c0886ce9593208f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 593.952397] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 5.966s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 593.954244] env[64020]: INFO nova.compute.claims [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 593.956101] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 262feed136cf42afb8b08acabc849c92 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 594.018448] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6c096701287d48278c0886ce9593208f [ 594.021139] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 262feed136cf42afb8b08acabc849c92 [ 594.063329] env[64020]: ERROR nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4c2e20e6-3e5d-4866-b20c-469a896012eb, please check neutron logs for more information. [ 594.063329] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 594.063329] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 594.063329] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 594.063329] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 594.063329] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 594.063329] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 594.063329] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 594.063329] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 594.063329] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 594.063329] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 594.063329] env[64020]: ERROR nova.compute.manager raise self.value [ 594.063329] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 594.063329] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 594.063329] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 594.063329] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 594.063853] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 594.063853] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 594.063853] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4c2e20e6-3e5d-4866-b20c-469a896012eb, please check neutron logs for more information. [ 594.063853] env[64020]: ERROR nova.compute.manager [ 594.063853] env[64020]: Traceback (most recent call last): [ 594.063853] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 594.063853] env[64020]: listener.cb(fileno) [ 594.063853] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 594.063853] env[64020]: result = function(*args, **kwargs) [ 594.063853] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 594.063853] env[64020]: return func(*args, **kwargs) [ 594.063853] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 594.063853] env[64020]: raise e [ 594.063853] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 594.063853] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 594.063853] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 594.063853] env[64020]: created_port_ids = self._update_ports_for_instance( [ 594.063853] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 594.063853] env[64020]: with excutils.save_and_reraise_exception(): [ 594.063853] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 594.063853] env[64020]: self.force_reraise() [ 594.063853] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 594.063853] env[64020]: raise self.value [ 594.063853] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 594.063853] env[64020]: updated_port = self._update_port( [ 594.063853] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 594.063853] env[64020]: _ensure_no_port_binding_failure(port) [ 594.063853] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 594.063853] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 594.064643] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 4c2e20e6-3e5d-4866-b20c-469a896012eb, please check neutron logs for more information. [ 594.064643] env[64020]: Removing descriptor: 15 [ 594.064643] env[64020]: ERROR nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4c2e20e6-3e5d-4866-b20c-469a896012eb, please check neutron logs for more information. [ 594.064643] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Traceback (most recent call last): [ 594.064643] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 594.064643] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] yield resources [ 594.064643] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 594.064643] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] self.driver.spawn(context, instance, image_meta, [ 594.064643] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 594.064643] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 594.064643] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 594.064643] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] vm_ref = self.build_virtual_machine(instance, [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] vif_infos = vmwarevif.get_vif_info(self._session, [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] for vif in network_info: [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] return self._sync_wrapper(fn, *args, **kwargs) [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] self.wait() [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] self[:] = self._gt.wait() [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] return self._exit_event.wait() [ 594.064964] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] result = hub.switch() [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] return self.greenlet.switch() [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] result = function(*args, **kwargs) [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] return func(*args, **kwargs) [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] raise e [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] nwinfo = self.network_api.allocate_for_instance( [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 594.065328] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] created_port_ids = self._update_ports_for_instance( [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] with excutils.save_and_reraise_exception(): [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] self.force_reraise() [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] raise self.value [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] updated_port = self._update_port( [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] _ensure_no_port_binding_failure(port) [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 594.065695] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] raise exception.PortBindingFailed(port_id=port['id']) [ 594.066024] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] nova.exception.PortBindingFailed: Binding failed for port 4c2e20e6-3e5d-4866-b20c-469a896012eb, please check neutron logs for more information. [ 594.066024] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] [ 594.066024] env[64020]: INFO nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Terminating instance [ 594.066893] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquiring lock "refresh_cache-4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 594.067037] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquired lock "refresh_cache-4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 594.067203] env[64020]: DEBUG nova.network.neutron [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 594.067599] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 0b33cc0e040e400f9f0b78f6da555916 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 594.068364] env[64020]: DEBUG nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 594.069920] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 3908636fe1b44669ae48e52febe3a73f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 594.074085] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b33cc0e040e400f9f0b78f6da555916 [ 594.108850] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3908636fe1b44669ae48e52febe3a73f [ 594.319181] env[64020]: DEBUG nova.network.neutron [-] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 594.319181] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3efa946ad46447f785eff51a71f06391 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 594.329077] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3efa946ad46447f785eff51a71f06391 [ 594.461303] env[64020]: DEBUG nova.compute.utils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 594.461303] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg c991d3a00b264262b54ed99a39a18e7b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 594.461303] env[64020]: DEBUG nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 594.461303] env[64020]: DEBUG nova.network.neutron [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 594.465282] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg a1aed46d4abc4f0b98e94ff8dfa0ec10 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 594.481425] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1aed46d4abc4f0b98e94ff8dfa0ec10 [ 594.482029] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c991d3a00b264262b54ed99a39a18e7b [ 594.552567] env[64020]: DEBUG nova.policy [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '87f7c1b2667d4613954fe14d9ffc28de', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5864c3d0f921416a915d9cd4ca5f9065', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 594.602198] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 594.634698] env[64020]: DEBUG nova.network.neutron [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 594.821080] env[64020]: INFO nova.compute.manager [-] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Took 1.08 seconds to deallocate network for instance. [ 594.824982] env[64020]: DEBUG nova.compute.claims [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 594.824982] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 594.861155] env[64020]: DEBUG nova.compute.manager [req-93419612-d20a-4a6b-bd49-08d476d4b44b req-e9aa54d0-2bae-458b-8493-bd14f533bff7 service nova] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Received event network-vif-deleted-e25137f1-b446-40b1-9e4b-0854030bbd51 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 594.951029] env[64020]: DEBUG nova.network.neutron [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 594.951743] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 80f72b1a67f542808a1aa68fb24dd820 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 594.965643] env[64020]: DEBUG nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 594.966990] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 1dd044bcc4d2405fbb4518e281d70dbb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 594.968165] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80f72b1a67f542808a1aa68fb24dd820 [ 595.136973] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1dd044bcc4d2405fbb4518e281d70dbb [ 595.174323] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1dbc7868-57c3-40d6-bf46-e947a5e09892 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.182547] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c705671f-18ac-4dee-9cc4-9990c02a00db {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.226250] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2812172-be84-4ef3-ac8f-1c2d4f5674ce {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.238161] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2fc1ab4-0a41-430d-b581-b39bdc87bbfc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.254691] env[64020]: DEBUG nova.compute.provider_tree [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 595.255223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 647181be10854617946d7e1fa646b019 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 595.266371] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 647181be10854617946d7e1fa646b019 [ 595.366562] env[64020]: DEBUG nova.network.neutron [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Successfully created port: 3d243fb0-6d5c-4646-b594-0f1677cf790f {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 595.459212] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Releasing lock "refresh_cache-4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 595.459701] env[64020]: DEBUG nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 595.459914] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 595.460269] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-46390d1b-37a7-43fb-9979-8ecefc903e75 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.470184] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b8fe7c4-22cd-4789-9bf4-a6b54a852425 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.487749] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 37d436860bc44178a632f990476d28b8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 595.500303] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2 could not be found. [ 595.500596] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 595.500813] env[64020]: INFO nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Took 0.04 seconds to destroy the instance on the hypervisor. [ 595.501206] env[64020]: DEBUG oslo.service.loopingcall [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 595.501366] env[64020]: DEBUG nova.compute.manager [-] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 595.501450] env[64020]: DEBUG nova.network.neutron [-] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 595.515138] env[64020]: DEBUG nova.compute.manager [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Received event network-vif-deleted-32bd22fb-e5de-4222-8ff3-1328a2144ded {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 595.515138] env[64020]: DEBUG nova.compute.manager [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Received event network-changed-06ebe7b0-7b82-4f26-85b9-cc69537dc66f {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 595.515138] env[64020]: DEBUG nova.compute.manager [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Refreshing instance network info cache due to event network-changed-06ebe7b0-7b82-4f26-85b9-cc69537dc66f. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 595.515138] env[64020]: DEBUG oslo_concurrency.lockutils [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] Acquiring lock "refresh_cache-c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 595.515138] env[64020]: DEBUG oslo_concurrency.lockutils [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] Acquired lock "refresh_cache-c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 595.515311] env[64020]: DEBUG nova.network.neutron [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Refreshing network info cache for port 06ebe7b0-7b82-4f26-85b9-cc69537dc66f {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 595.515311] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] Expecting reply to msg 1ca16f0502cf4d3ebad60a3474f950af in queue reply_57893177120949e6a93cb88e15cd42b4 [ 595.524749] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ca16f0502cf4d3ebad60a3474f950af [ 595.543768] env[64020]: DEBUG nova.network.neutron [-] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 595.544053] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a568663ba6714d57b35510ef6f030745 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 595.573691] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a568663ba6714d57b35510ef6f030745 [ 595.578128] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37d436860bc44178a632f990476d28b8 [ 595.758341] env[64020]: DEBUG nova.scheduler.client.report [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 595.760895] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg ef54414d0af742f69efe7343c6bc6b7c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 595.776495] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef54414d0af742f69efe7343c6bc6b7c [ 595.990955] env[64020]: DEBUG nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 596.015774] env[64020]: DEBUG nova.virt.hardware [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 596.016013] env[64020]: DEBUG nova.virt.hardware [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 596.016176] env[64020]: DEBUG nova.virt.hardware [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 596.016353] env[64020]: DEBUG nova.virt.hardware [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 596.016494] env[64020]: DEBUG nova.virt.hardware [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 596.016637] env[64020]: DEBUG nova.virt.hardware [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 596.016840] env[64020]: DEBUG nova.virt.hardware [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 596.017021] env[64020]: DEBUG nova.virt.hardware [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 596.017137] env[64020]: DEBUG nova.virt.hardware [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 596.017294] env[64020]: DEBUG nova.virt.hardware [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 596.017451] env[64020]: DEBUG nova.virt.hardware [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 596.020115] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb5a2084-aa5d-4235-a5b1-f057b1dea860 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 596.028250] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b732fee-131c-4a8b-a3d4-89924f7b054e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 596.046375] env[64020]: DEBUG nova.network.neutron [-] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 596.046825] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 596698a0045f41d6afb6b4603b6dfde6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 596.058901] env[64020]: DEBUG nova.network.neutron [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 596.058901] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 596698a0045f41d6afb6b4603b6dfde6 [ 596.065416] env[64020]: ERROR nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3a25c429-30d6-43fb-927d-555318c582a6, please check neutron logs for more information. [ 596.065416] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 596.065416] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 596.065416] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 596.065416] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 596.065416] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 596.065416] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 596.065416] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 596.065416] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 596.065416] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 596.065416] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 596.065416] env[64020]: ERROR nova.compute.manager raise self.value [ 596.065416] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 596.065416] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 596.065416] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 596.065416] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 596.065901] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 596.065901] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 596.065901] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3a25c429-30d6-43fb-927d-555318c582a6, please check neutron logs for more information. [ 596.065901] env[64020]: ERROR nova.compute.manager [ 596.066770] env[64020]: Traceback (most recent call last): [ 596.066770] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 596.066770] env[64020]: listener.cb(fileno) [ 596.066770] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 596.066770] env[64020]: result = function(*args, **kwargs) [ 596.066770] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 596.066770] env[64020]: return func(*args, **kwargs) [ 596.066770] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 596.066770] env[64020]: raise e [ 596.066770] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 596.066770] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 596.066770] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 596.066770] env[64020]: created_port_ids = self._update_ports_for_instance( [ 596.066770] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 596.066770] env[64020]: with excutils.save_and_reraise_exception(): [ 596.066770] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 596.066770] env[64020]: self.force_reraise() [ 596.066770] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 596.066770] env[64020]: raise self.value [ 596.066770] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 596.066770] env[64020]: updated_port = self._update_port( [ 596.066770] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 596.066770] env[64020]: _ensure_no_port_binding_failure(port) [ 596.066770] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 596.066770] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 596.066770] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 3a25c429-30d6-43fb-927d-555318c582a6, please check neutron logs for more information. [ 596.066770] env[64020]: Removing descriptor: 16 [ 596.069046] env[64020]: ERROR nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3a25c429-30d6-43fb-927d-555318c582a6, please check neutron logs for more information. [ 596.069046] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Traceback (most recent call last): [ 596.069046] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 596.069046] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] yield resources [ 596.069046] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 596.069046] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] self.driver.spawn(context, instance, image_meta, [ 596.069046] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 596.069046] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 596.069046] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 596.069046] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] vm_ref = self.build_virtual_machine(instance, [ 596.069046] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] vif_infos = vmwarevif.get_vif_info(self._session, [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] for vif in network_info: [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] return self._sync_wrapper(fn, *args, **kwargs) [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] self.wait() [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] self[:] = self._gt.wait() [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] return self._exit_event.wait() [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 596.069602] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] result = hub.switch() [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] return self.greenlet.switch() [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] result = function(*args, **kwargs) [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] return func(*args, **kwargs) [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] raise e [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] nwinfo = self.network_api.allocate_for_instance( [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] created_port_ids = self._update_ports_for_instance( [ 596.070235] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] with excutils.save_and_reraise_exception(): [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] self.force_reraise() [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] raise self.value [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] updated_port = self._update_port( [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] _ensure_no_port_binding_failure(port) [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] raise exception.PortBindingFailed(port_id=port['id']) [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] nova.exception.PortBindingFailed: Binding failed for port 3a25c429-30d6-43fb-927d-555318c582a6, please check neutron logs for more information. [ 596.070789] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] [ 596.071386] env[64020]: INFO nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Terminating instance [ 596.073232] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "refresh_cache-60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 596.073614] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquired lock "refresh_cache-60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 596.073911] env[64020]: DEBUG nova.network.neutron [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 596.074434] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 7aed543ab2054b358b33db4a3a9e3041 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 596.082910] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7aed543ab2054b358b33db4a3a9e3041 [ 596.209088] env[64020]: DEBUG nova.network.neutron [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 596.209567] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] Expecting reply to msg a3c2e5ae97184320a23cb92f4af55bf2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 596.217185] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a3c2e5ae97184320a23cb92f4af55bf2 [ 596.250121] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquiring lock "e5b12149-0e4c-41b2-b726-f8a55452669a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 596.250349] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "e5b12149-0e4c-41b2-b726-f8a55452669a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.264284] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.312s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 596.264782] env[64020]: DEBUG nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 596.266431] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg dc7f41c376c64d0cbb0d38f4598c02af in queue reply_57893177120949e6a93cb88e15cd42b4 [ 596.272024] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 7.550s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.272024] env[64020]: INFO nova.compute.claims [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 596.272024] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 32f4f2c3d1814df5b20837c08611cf60 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 596.308739] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc7f41c376c64d0cbb0d38f4598c02af [ 596.314164] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 32f4f2c3d1814df5b20837c08611cf60 [ 596.550228] env[64020]: INFO nova.compute.manager [-] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Took 1.05 seconds to deallocate network for instance. [ 596.551978] env[64020]: DEBUG nova.compute.claims [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 596.551978] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 596.604665] env[64020]: DEBUG nova.network.neutron [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 596.713982] env[64020]: DEBUG oslo_concurrency.lockutils [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] Releasing lock "refresh_cache-c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 596.714237] env[64020]: DEBUG nova.compute.manager [req-e7f0171b-13f3-4c7b-9bc6-9b3b2827021e req-d6398d3d-7244-4476-a276-79ae54a297f2 service nova] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Received event network-vif-deleted-06ebe7b0-7b82-4f26-85b9-cc69537dc66f {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 596.773290] env[64020]: DEBUG nova.compute.utils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 596.773941] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg ba1698822b9548d2976a8cd828e2a964 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 596.774893] env[64020]: DEBUG nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 596.775061] env[64020]: DEBUG nova.network.neutron [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 596.778122] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 35d5372361254b23b2fc4e9381cf1ebb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 596.790723] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba1698822b9548d2976a8cd828e2a964 [ 596.791313] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35d5372361254b23b2fc4e9381cf1ebb [ 596.807308] env[64020]: DEBUG nova.network.neutron [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 596.807402] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 3cbcc0b4e5c04d72aac09d828768f26e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 596.822737] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3cbcc0b4e5c04d72aac09d828768f26e [ 596.903283] env[64020]: DEBUG nova.policy [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4c462a89ea774142a0e669b5acbce371', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '97a2fdceb396470ba1ece4bfd7d002e5', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 597.076254] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Acquiring lock "a98ba289-b065-4cef-a7bd-7b7cd07b141d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 597.076472] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Lock "a98ba289-b065-4cef-a7bd-7b7cd07b141d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 597.280984] env[64020]: DEBUG nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 597.282975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg c8d9f31a66db4d7791f9b66d081cdef7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 597.320439] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Releasing lock "refresh_cache-60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 597.320439] env[64020]: DEBUG nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 597.320439] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 597.320439] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-440ae968-ff17-49cb-9dcf-d61302f1e711 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.330387] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-feac7238-5616-437f-9fd8-9455dcc44b89 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.350687] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c8d9f31a66db4d7791f9b66d081cdef7 [ 597.364281] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3 could not be found. [ 597.364502] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 597.364726] env[64020]: INFO nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Took 0.05 seconds to destroy the instance on the hypervisor. [ 597.364912] env[64020]: DEBUG oslo.service.loopingcall [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 597.365129] env[64020]: DEBUG nova.compute.manager [-] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 597.365220] env[64020]: DEBUG nova.network.neutron [-] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 597.388091] env[64020]: DEBUG nova.network.neutron [-] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 597.388604] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8ddfbc01fc6143d68625db0c957df1bb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 597.396716] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ddfbc01fc6143d68625db0c957df1bb [ 597.534636] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36a04599-de83-442e-b260-9c0625dcd960 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.542593] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c574e6f1-db4a-48c0-a2dd-ffa78ef1eee8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.572427] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70dcad74-b389-4e35-8a13-384351c0d803 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.579814] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94fd2e03-d63e-4806-93a6-aa54fb0b4847 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.596231] env[64020]: DEBUG nova.compute.provider_tree [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 597.596890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg f6e61792bd3f459caedc64fda758f418 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 597.604712] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f6e61792bd3f459caedc64fda758f418 [ 597.646758] env[64020]: DEBUG nova.compute.manager [req-674adf76-dd0c-4dbd-a120-3ac0e8dc06ae req-db82cadb-0706-4e68-95cd-53415eb58f2d service nova] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Received event network-changed-4c2e20e6-3e5d-4866-b20c-469a896012eb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 597.646941] env[64020]: DEBUG nova.compute.manager [req-674adf76-dd0c-4dbd-a120-3ac0e8dc06ae req-db82cadb-0706-4e68-95cd-53415eb58f2d service nova] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Refreshing instance network info cache due to event network-changed-4c2e20e6-3e5d-4866-b20c-469a896012eb. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 597.647794] env[64020]: DEBUG oslo_concurrency.lockutils [req-674adf76-dd0c-4dbd-a120-3ac0e8dc06ae req-db82cadb-0706-4e68-95cd-53415eb58f2d service nova] Acquiring lock "refresh_cache-4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 597.647794] env[64020]: DEBUG oslo_concurrency.lockutils [req-674adf76-dd0c-4dbd-a120-3ac0e8dc06ae req-db82cadb-0706-4e68-95cd-53415eb58f2d service nova] Acquired lock "refresh_cache-4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 597.647794] env[64020]: DEBUG nova.network.neutron [req-674adf76-dd0c-4dbd-a120-3ac0e8dc06ae req-db82cadb-0706-4e68-95cd-53415eb58f2d service nova] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Refreshing network info cache for port 4c2e20e6-3e5d-4866-b20c-469a896012eb {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 597.647794] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-674adf76-dd0c-4dbd-a120-3ac0e8dc06ae req-db82cadb-0706-4e68-95cd-53415eb58f2d service nova] Expecting reply to msg 847c0a34a6d84519bbdcaaf51d1fe9a7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 597.655265] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 847c0a34a6d84519bbdcaaf51d1fe9a7 [ 597.691686] env[64020]: DEBUG nova.network.neutron [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Successfully created port: 5820ea5a-bebc-46c4-a6a9-3551052bf734 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 597.789711] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 7dde9c6178284bffbaa3873ef2aff005 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 597.840992] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7dde9c6178284bffbaa3873ef2aff005 [ 597.894526] env[64020]: DEBUG nova.network.neutron [-] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 597.894978] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 11a73bcbb32b41a79aca24176ca2ed44 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 597.905342] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 11a73bcbb32b41a79aca24176ca2ed44 [ 598.099933] env[64020]: DEBUG nova.scheduler.client.report [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 598.103004] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 792f49e1f11246edbf7788bdabe50fc4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 598.118214] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 792f49e1f11246edbf7788bdabe50fc4 [ 598.188777] env[64020]: DEBUG nova.network.neutron [req-674adf76-dd0c-4dbd-a120-3ac0e8dc06ae req-db82cadb-0706-4e68-95cd-53415eb58f2d service nova] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 598.293353] env[64020]: DEBUG nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 598.315709] env[64020]: DEBUG nova.virt.hardware [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 598.315940] env[64020]: DEBUG nova.virt.hardware [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 598.316257] env[64020]: DEBUG nova.virt.hardware [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 598.316385] env[64020]: DEBUG nova.virt.hardware [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 598.316438] env[64020]: DEBUG nova.virt.hardware [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 598.317190] env[64020]: DEBUG nova.virt.hardware [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 598.317190] env[64020]: DEBUG nova.virt.hardware [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 598.317190] env[64020]: DEBUG nova.virt.hardware [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 598.317190] env[64020]: DEBUG nova.virt.hardware [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 598.317314] env[64020]: DEBUG nova.virt.hardware [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 598.317716] env[64020]: DEBUG nova.virt.hardware [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 598.318442] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16400e98-4608-4cbf-a01c-341bcc8e83e8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.326756] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4286d30-c757-4ba8-96e3-cfcee9e0f2a2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.351512] env[64020]: DEBUG nova.network.neutron [req-674adf76-dd0c-4dbd-a120-3ac0e8dc06ae req-db82cadb-0706-4e68-95cd-53415eb58f2d service nova] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 598.351966] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-674adf76-dd0c-4dbd-a120-3ac0e8dc06ae req-db82cadb-0706-4e68-95cd-53415eb58f2d service nova] Expecting reply to msg 2d29a534172f40c5ba7c9ab1a86126db in queue reply_57893177120949e6a93cb88e15cd42b4 [ 598.364788] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d29a534172f40c5ba7c9ab1a86126db [ 598.400694] env[64020]: INFO nova.compute.manager [-] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Took 1.03 seconds to deallocate network for instance. [ 598.400694] env[64020]: DEBUG nova.compute.claims [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 598.400694] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 598.482733] env[64020]: DEBUG nova.compute.manager [req-4ff05a6d-509f-4c5f-baf1-113ed52fb863 req-f80840cd-3968-4143-b446-450ebe499246 service nova] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Received event network-changed-3a25c429-30d6-43fb-927d-555318c582a6 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 598.482777] env[64020]: DEBUG nova.compute.manager [req-4ff05a6d-509f-4c5f-baf1-113ed52fb863 req-f80840cd-3968-4143-b446-450ebe499246 service nova] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Refreshing instance network info cache due to event network-changed-3a25c429-30d6-43fb-927d-555318c582a6. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 598.482965] env[64020]: DEBUG oslo_concurrency.lockutils [req-4ff05a6d-509f-4c5f-baf1-113ed52fb863 req-f80840cd-3968-4143-b446-450ebe499246 service nova] Acquiring lock "refresh_cache-60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 598.483112] env[64020]: DEBUG oslo_concurrency.lockutils [req-4ff05a6d-509f-4c5f-baf1-113ed52fb863 req-f80840cd-3968-4143-b446-450ebe499246 service nova] Acquired lock "refresh_cache-60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 598.483268] env[64020]: DEBUG nova.network.neutron [req-4ff05a6d-509f-4c5f-baf1-113ed52fb863 req-f80840cd-3968-4143-b446-450ebe499246 service nova] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Refreshing network info cache for port 3a25c429-30d6-43fb-927d-555318c582a6 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 598.483704] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-4ff05a6d-509f-4c5f-baf1-113ed52fb863 req-f80840cd-3968-4143-b446-450ebe499246 service nova] Expecting reply to msg 0dbb64ab79cb477980afb41e3fae11a6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 598.490363] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0dbb64ab79cb477980afb41e3fae11a6 [ 598.605784] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.338s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 598.606826] env[64020]: DEBUG nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 598.607998] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 510563b56e9b4e82a15cea3c324f8e25 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 598.609011] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 6.493s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 598.615921] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 598.615921] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=64020) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 598.615921] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 6.226s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 598.615921] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 09424f270f914bc8ad4adeecd5ca0666 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 598.615921] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd8e195b-3546-469b-96a9-3561bbd0bc1c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.628976] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60fa1c99-ac6c-4716-ac78-6186775c710d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.648936] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17a48f57-4baf-40a7-8ad0-928eddc2797e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.655637] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9673c5f-a6a5-4207-865c-f03f5f630c97 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.687731] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181806MB free_disk=120GB free_vcpus=48 pci_devices=None {{(pid=64020) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 598.687731] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 598.689294] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 510563b56e9b4e82a15cea3c324f8e25 [ 598.689294] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 09424f270f914bc8ad4adeecd5ca0666 [ 598.865511] env[64020]: DEBUG oslo_concurrency.lockutils [req-674adf76-dd0c-4dbd-a120-3ac0e8dc06ae req-db82cadb-0706-4e68-95cd-53415eb58f2d service nova] Releasing lock "refresh_cache-4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 598.865511] env[64020]: DEBUG nova.compute.manager [req-674adf76-dd0c-4dbd-a120-3ac0e8dc06ae req-db82cadb-0706-4e68-95cd-53415eb58f2d service nova] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Received event network-vif-deleted-4c2e20e6-3e5d-4866-b20c-469a896012eb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 598.897357] env[64020]: ERROR nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3d243fb0-6d5c-4646-b594-0f1677cf790f, please check neutron logs for more information. [ 598.897357] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 598.897357] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 598.897357] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 598.897357] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 598.897357] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 598.897357] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 598.897357] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 598.897357] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 598.897357] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 598.897357] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 598.897357] env[64020]: ERROR nova.compute.manager raise self.value [ 598.897357] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 598.897357] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 598.897357] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 598.897357] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 598.897865] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 598.897865] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 598.897865] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3d243fb0-6d5c-4646-b594-0f1677cf790f, please check neutron logs for more information. [ 598.897865] env[64020]: ERROR nova.compute.manager [ 598.897865] env[64020]: Traceback (most recent call last): [ 598.897865] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 598.897865] env[64020]: listener.cb(fileno) [ 598.897865] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 598.897865] env[64020]: result = function(*args, **kwargs) [ 598.897865] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 598.897865] env[64020]: return func(*args, **kwargs) [ 598.897865] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 598.897865] env[64020]: raise e [ 598.897865] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 598.897865] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 598.897865] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 598.897865] env[64020]: created_port_ids = self._update_ports_for_instance( [ 598.897865] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 598.897865] env[64020]: with excutils.save_and_reraise_exception(): [ 598.897865] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 598.897865] env[64020]: self.force_reraise() [ 598.897865] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 598.897865] env[64020]: raise self.value [ 598.897865] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 598.897865] env[64020]: updated_port = self._update_port( [ 598.897865] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 598.897865] env[64020]: _ensure_no_port_binding_failure(port) [ 598.897865] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 598.897865] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 598.898724] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 3d243fb0-6d5c-4646-b594-0f1677cf790f, please check neutron logs for more information. [ 598.898724] env[64020]: Removing descriptor: 15 [ 598.898724] env[64020]: ERROR nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3d243fb0-6d5c-4646-b594-0f1677cf790f, please check neutron logs for more information. [ 598.898724] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Traceback (most recent call last): [ 598.898724] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 598.898724] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] yield resources [ 598.898724] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 598.898724] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] self.driver.spawn(context, instance, image_meta, [ 598.898724] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 598.898724] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 598.898724] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 598.898724] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] vm_ref = self.build_virtual_machine(instance, [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] vif_infos = vmwarevif.get_vif_info(self._session, [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] for vif in network_info: [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] return self._sync_wrapper(fn, *args, **kwargs) [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] self.wait() [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] self[:] = self._gt.wait() [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] return self._exit_event.wait() [ 598.899180] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] result = hub.switch() [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] return self.greenlet.switch() [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] result = function(*args, **kwargs) [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] return func(*args, **kwargs) [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] raise e [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] nwinfo = self.network_api.allocate_for_instance( [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 598.899599] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] created_port_ids = self._update_ports_for_instance( [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] with excutils.save_and_reraise_exception(): [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] self.force_reraise() [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] raise self.value [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] updated_port = self._update_port( [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] _ensure_no_port_binding_failure(port) [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 598.900115] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] raise exception.PortBindingFailed(port_id=port['id']) [ 598.900484] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] nova.exception.PortBindingFailed: Binding failed for port 3d243fb0-6d5c-4646-b594-0f1677cf790f, please check neutron logs for more information. [ 598.900484] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] [ 598.900484] env[64020]: INFO nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Terminating instance [ 598.902591] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Acquiring lock "refresh_cache-b18fca61-2b17-4758-824d-e02d2cb342d8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 598.902591] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Acquired lock "refresh_cache-b18fca61-2b17-4758-824d-e02d2cb342d8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 598.902591] env[64020]: DEBUG nova.network.neutron [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 598.902591] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 2f735e7b841845e584e6f9c3d694a845 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 598.911256] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f735e7b841845e584e6f9c3d694a845 [ 599.032266] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquiring lock "ae57ce0e-46b5-4490-af3f-4b2537bf316a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 599.032601] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "ae57ce0e-46b5-4490-af3f-4b2537bf316a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 599.034490] env[64020]: DEBUG nova.network.neutron [req-4ff05a6d-509f-4c5f-baf1-113ed52fb863 req-f80840cd-3968-4143-b446-450ebe499246 service nova] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 599.115244] env[64020]: DEBUG nova.compute.utils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 599.115885] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg c1f8cf2dc9874fd3bb54206bf3a56b2a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 599.116834] env[64020]: DEBUG nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 599.123142] env[64020]: DEBUG nova.network.neutron [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 599.132230] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c1f8cf2dc9874fd3bb54206bf3a56b2a [ 599.275247] env[64020]: DEBUG nova.policy [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '92dbbb520a274d98baff4fb9a10c6a0d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'dcb43e39f8964efb9dde2ad826a224d4', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 599.363550] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac9e2f1e-002b-45b8-bea9-e938b573566c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.371382] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28c4e643-5849-4a83-b75e-9891052f27a9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.406804] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30c8fcb7-16d6-4a86-a52e-c04d67c5d2b7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.415225] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21df4374-7340-4d91-b9a7-4130ec6ca26b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.428747] env[64020]: DEBUG nova.compute.provider_tree [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 599.429326] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 134526cabe354a9f81ef5ab2c40bb6b0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 599.436633] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 134526cabe354a9f81ef5ab2c40bb6b0 [ 599.510568] env[64020]: DEBUG nova.network.neutron [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 599.593274] env[64020]: DEBUG nova.network.neutron [req-4ff05a6d-509f-4c5f-baf1-113ed52fb863 req-f80840cd-3968-4143-b446-450ebe499246 service nova] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 599.593274] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-4ff05a6d-509f-4c5f-baf1-113ed52fb863 req-f80840cd-3968-4143-b446-450ebe499246 service nova] Expecting reply to msg 3462ca6dbcac4090a6a550693f2ac672 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 599.601288] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3462ca6dbcac4090a6a550693f2ac672 [ 599.622397] env[64020]: DEBUG nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 599.622397] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 110f4186525b4a5a8a0f840b4532753a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 599.659925] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 110f4186525b4a5a8a0f840b4532753a [ 599.832716] env[64020]: DEBUG nova.network.neutron [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 599.833415] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg fcd3c6d342294e50904acb88d14a8afa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 599.843242] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fcd3c6d342294e50904acb88d14a8afa [ 599.934838] env[64020]: DEBUG nova.scheduler.client.report [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 599.934838] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg f33910c3a2ff4941899d375847b58d3c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 599.946276] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f33910c3a2ff4941899d375847b58d3c [ 599.959000] env[64020]: ERROR nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 5820ea5a-bebc-46c4-a6a9-3551052bf734, please check neutron logs for more information. [ 599.959000] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 599.959000] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 599.959000] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 599.959000] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 599.959000] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 599.959000] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 599.959000] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 599.959000] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 599.959000] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 599.959000] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 599.959000] env[64020]: ERROR nova.compute.manager raise self.value [ 599.959000] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 599.959000] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 599.959000] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 599.959000] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 599.959482] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 599.959482] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 599.959482] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 5820ea5a-bebc-46c4-a6a9-3551052bf734, please check neutron logs for more information. [ 599.959482] env[64020]: ERROR nova.compute.manager [ 599.960517] env[64020]: Traceback (most recent call last): [ 599.960517] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 599.960517] env[64020]: listener.cb(fileno) [ 599.960517] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 599.960517] env[64020]: result = function(*args, **kwargs) [ 599.960517] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 599.960517] env[64020]: return func(*args, **kwargs) [ 599.960517] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 599.960517] env[64020]: raise e [ 599.960517] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 599.960517] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 599.960517] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 599.960517] env[64020]: created_port_ids = self._update_ports_for_instance( [ 599.960517] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 599.960517] env[64020]: with excutils.save_and_reraise_exception(): [ 599.960517] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 599.960517] env[64020]: self.force_reraise() [ 599.960517] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 599.960517] env[64020]: raise self.value [ 599.960517] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 599.960517] env[64020]: updated_port = self._update_port( [ 599.960517] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 599.960517] env[64020]: _ensure_no_port_binding_failure(port) [ 599.960517] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 599.960517] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 599.960517] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 5820ea5a-bebc-46c4-a6a9-3551052bf734, please check neutron logs for more information. [ 599.960517] env[64020]: Removing descriptor: 16 [ 599.962126] env[64020]: ERROR nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 5820ea5a-bebc-46c4-a6a9-3551052bf734, please check neutron logs for more information. [ 599.962126] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Traceback (most recent call last): [ 599.962126] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 599.962126] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] yield resources [ 599.962126] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 599.962126] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] self.driver.spawn(context, instance, image_meta, [ 599.962126] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 599.962126] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 599.962126] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 599.962126] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] vm_ref = self.build_virtual_machine(instance, [ 599.962126] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] vif_infos = vmwarevif.get_vif_info(self._session, [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] for vif in network_info: [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] return self._sync_wrapper(fn, *args, **kwargs) [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] self.wait() [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] self[:] = self._gt.wait() [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] return self._exit_event.wait() [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 599.962480] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] result = hub.switch() [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] return self.greenlet.switch() [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] result = function(*args, **kwargs) [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] return func(*args, **kwargs) [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] raise e [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] nwinfo = self.network_api.allocate_for_instance( [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] created_port_ids = self._update_ports_for_instance( [ 599.962818] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] with excutils.save_and_reraise_exception(): [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] self.force_reraise() [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] raise self.value [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] updated_port = self._update_port( [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] _ensure_no_port_binding_failure(port) [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] raise exception.PortBindingFailed(port_id=port['id']) [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] nova.exception.PortBindingFailed: Binding failed for port 5820ea5a-bebc-46c4-a6a9-3551052bf734, please check neutron logs for more information. [ 599.963153] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] [ 599.963564] env[64020]: INFO nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Terminating instance [ 599.969344] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Acquiring lock "refresh_cache-7529a478-7efe-4dff-8429-810549f56bd7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 599.969344] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Acquired lock "refresh_cache-7529a478-7efe-4dff-8429-810549f56bd7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 599.969344] env[64020]: DEBUG nova.network.neutron [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 599.969344] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg a8d3a1504a5847ec9b1aea4854485079 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 599.986015] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8d3a1504a5847ec9b1aea4854485079 [ 600.094992] env[64020]: DEBUG oslo_concurrency.lockutils [req-4ff05a6d-509f-4c5f-baf1-113ed52fb863 req-f80840cd-3968-4143-b446-450ebe499246 service nova] Releasing lock "refresh_cache-60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 600.095307] env[64020]: DEBUG nova.compute.manager [req-4ff05a6d-509f-4c5f-baf1-113ed52fb863 req-f80840cd-3968-4143-b446-450ebe499246 service nova] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Received event network-vif-deleted-3a25c429-30d6-43fb-927d-555318c582a6 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 600.126962] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 7f2fce0d7d8440faa90060553d8f4816 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 600.174963] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f2fce0d7d8440faa90060553d8f4816 [ 600.286632] env[64020]: DEBUG nova.network.neutron [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Successfully created port: b92962a4-4c4a-4ade-9372-98c32103ec91 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 600.340823] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Releasing lock "refresh_cache-b18fca61-2b17-4758-824d-e02d2cb342d8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 600.341247] env[64020]: DEBUG nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 600.341437] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 600.341737] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5517fdcf-0475-4034-b868-5acc7da56b8a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.351200] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3427ae39-68ed-4e11-8c5f-3179c5f3d002 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.375300] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquiring lock "76de4647-9281-4c49-b4f1-16f0ea2ff313" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 600.375544] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "76de4647-9281-4c49-b4f1-16f0ea2ff313" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 600.385266] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b18fca61-2b17-4758-824d-e02d2cb342d8 could not be found. [ 600.385478] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 600.385654] env[64020]: INFO nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Took 0.04 seconds to destroy the instance on the hypervisor. [ 600.385902] env[64020]: DEBUG oslo.service.loopingcall [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 600.386103] env[64020]: DEBUG nova.compute.manager [-] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 600.386196] env[64020]: DEBUG nova.network.neutron [-] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 600.396311] env[64020]: DEBUG nova.compute.manager [req-a40f7dc4-b401-477b-a0d3-62a8f7c0d3df req-7a86136b-02a9-497c-a910-96bb1171f0ae service nova] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Received event network-changed-3d243fb0-6d5c-4646-b594-0f1677cf790f {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 600.396952] env[64020]: DEBUG nova.compute.manager [req-a40f7dc4-b401-477b-a0d3-62a8f7c0d3df req-7a86136b-02a9-497c-a910-96bb1171f0ae service nova] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Refreshing instance network info cache due to event network-changed-3d243fb0-6d5c-4646-b594-0f1677cf790f. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 600.397163] env[64020]: DEBUG oslo_concurrency.lockutils [req-a40f7dc4-b401-477b-a0d3-62a8f7c0d3df req-7a86136b-02a9-497c-a910-96bb1171f0ae service nova] Acquiring lock "refresh_cache-b18fca61-2b17-4758-824d-e02d2cb342d8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 600.397380] env[64020]: DEBUG oslo_concurrency.lockutils [req-a40f7dc4-b401-477b-a0d3-62a8f7c0d3df req-7a86136b-02a9-497c-a910-96bb1171f0ae service nova] Acquired lock "refresh_cache-b18fca61-2b17-4758-824d-e02d2cb342d8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 600.397613] env[64020]: DEBUG nova.network.neutron [req-a40f7dc4-b401-477b-a0d3-62a8f7c0d3df req-7a86136b-02a9-497c-a910-96bb1171f0ae service nova] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Refreshing network info cache for port 3d243fb0-6d5c-4646-b594-0f1677cf790f {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 600.398066] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-a40f7dc4-b401-477b-a0d3-62a8f7c0d3df req-7a86136b-02a9-497c-a910-96bb1171f0ae service nova] Expecting reply to msg ec99a50af37b48c58679ff3105978cac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 600.405195] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec99a50af37b48c58679ff3105978cac [ 600.413877] env[64020]: DEBUG nova.network.neutron [-] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 600.414781] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a780a4f34ec64e679f2e2958765f5e5d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 600.432717] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a780a4f34ec64e679f2e2958765f5e5d [ 600.437404] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.826s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 600.438022] env[64020]: ERROR nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e25137f1-b446-40b1-9e4b-0854030bbd51, please check neutron logs for more information. [ 600.438022] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Traceback (most recent call last): [ 600.438022] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 600.438022] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] self.driver.spawn(context, instance, image_meta, [ 600.438022] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 600.438022] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] self._vmops.spawn(context, instance, image_meta, injected_files, [ 600.438022] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 600.438022] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] vm_ref = self.build_virtual_machine(instance, [ 600.438022] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 600.438022] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] vif_infos = vmwarevif.get_vif_info(self._session, [ 600.438022] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] for vif in network_info: [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] return self._sync_wrapper(fn, *args, **kwargs) [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] self.wait() [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] self[:] = self._gt.wait() [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] return self._exit_event.wait() [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] result = hub.switch() [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 600.438415] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] return self.greenlet.switch() [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] result = function(*args, **kwargs) [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] return func(*args, **kwargs) [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] raise e [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] nwinfo = self.network_api.allocate_for_instance( [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] created_port_ids = self._update_ports_for_instance( [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] with excutils.save_and_reraise_exception(): [ 600.438726] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 600.439052] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] self.force_reraise() [ 600.439052] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 600.439052] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] raise self.value [ 600.439052] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 600.439052] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] updated_port = self._update_port( [ 600.439052] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 600.439052] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] _ensure_no_port_binding_failure(port) [ 600.439052] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 600.439052] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] raise exception.PortBindingFailed(port_id=port['id']) [ 600.439052] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] nova.exception.PortBindingFailed: Binding failed for port e25137f1-b446-40b1-9e4b-0854030bbd51, please check neutron logs for more information. [ 600.439052] env[64020]: ERROR nova.compute.manager [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] [ 600.439360] env[64020]: DEBUG nova.compute.utils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Binding failed for port e25137f1-b446-40b1-9e4b-0854030bbd51, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 600.441103] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 6.904s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 600.442860] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg 165c7378dc2a4ad3bcb12f76be1a55ba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 600.449434] env[64020]: DEBUG nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Build of instance 9d4a7822-643e-4bdb-b431-a16e5d183100 was re-scheduled: Binding failed for port e25137f1-b446-40b1-9e4b-0854030bbd51, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 600.449955] env[64020]: DEBUG nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 600.450223] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Acquiring lock "refresh_cache-9d4a7822-643e-4bdb-b431-a16e5d183100" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 600.450447] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Acquired lock "refresh_cache-9d4a7822-643e-4bdb-b431-a16e5d183100" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 600.450658] env[64020]: DEBUG nova.network.neutron [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 600.451093] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg d129224477c44cddb6da03af67751bc9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 600.458061] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d129224477c44cddb6da03af67751bc9 [ 600.485841] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 165c7378dc2a4ad3bcb12f76be1a55ba [ 600.517752] env[64020]: DEBUG nova.network.neutron [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 600.630647] env[64020]: DEBUG nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 600.663997] env[64020]: DEBUG nova.virt.hardware [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 600.664313] env[64020]: DEBUG nova.virt.hardware [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 600.664494] env[64020]: DEBUG nova.virt.hardware [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 600.664585] env[64020]: DEBUG nova.virt.hardware [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 600.664726] env[64020]: DEBUG nova.virt.hardware [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 600.664871] env[64020]: DEBUG nova.virt.hardware [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 600.665079] env[64020]: DEBUG nova.virt.hardware [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 600.665350] env[64020]: DEBUG nova.virt.hardware [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 600.665412] env[64020]: DEBUG nova.virt.hardware [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 600.665551] env[64020]: DEBUG nova.virt.hardware [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 600.665717] env[64020]: DEBUG nova.virt.hardware [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 600.666881] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec334786-df65-4039-9c39-ca8c8fbbd540 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.677430] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14ce2eac-ca4b-4b80-a2da-42fe2a50d305 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.772656] env[64020]: DEBUG nova.network.neutron [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 600.773347] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 2bfbf634ce924c73b952c32b9689538b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 600.784909] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2bfbf634ce924c73b952c32b9689538b [ 600.916734] env[64020]: DEBUG nova.network.neutron [-] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 600.917198] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bb220d4bca064558b4bfa97a7fabaa6a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 600.930047] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb220d4bca064558b4bfa97a7fabaa6a [ 600.931185] env[64020]: DEBUG nova.network.neutron [req-a40f7dc4-b401-477b-a0d3-62a8f7c0d3df req-7a86136b-02a9-497c-a910-96bb1171f0ae service nova] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 600.990813] env[64020]: DEBUG nova.network.neutron [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 601.148173] env[64020]: DEBUG nova.network.neutron [req-a40f7dc4-b401-477b-a0d3-62a8f7c0d3df req-7a86136b-02a9-497c-a910-96bb1171f0ae service nova] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 601.148745] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-a40f7dc4-b401-477b-a0d3-62a8f7c0d3df req-7a86136b-02a9-497c-a910-96bb1171f0ae service nova] Expecting reply to msg 9d5afdd734694bc3a858fe05e624c206 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 601.161359] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9d5afdd734694bc3a858fe05e624c206 [ 601.245287] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9442c7ce-2c2f-4318-954b-b92e6e8768e0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.255161] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0bb03b45-25bf-4a0d-8a49-bb9b9cc63502 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.288561] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Releasing lock "refresh_cache-7529a478-7efe-4dff-8429-810549f56bd7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 601.288992] env[64020]: DEBUG nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 601.289170] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 601.289685] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-38040587-33fe-4750-bf2d-dcbc225bffba {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.292070] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96c666da-2212-43a8-9c6b-235b49b7fcfa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.320706] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3eae923-b4ae-4a90-a967-1fd734e5d704 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.334429] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d064d466-c850-4791-a2ac-7e35517c51dc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.343227] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7529a478-7efe-4dff-8429-810549f56bd7 could not be found. [ 601.343458] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 601.343643] env[64020]: INFO nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Took 0.05 seconds to destroy the instance on the hypervisor. [ 601.343884] env[64020]: DEBUG oslo.service.loopingcall [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 601.351378] env[64020]: DEBUG nova.compute.manager [-] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 601.351483] env[64020]: DEBUG nova.network.neutron [-] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 601.354017] env[64020]: DEBUG nova.compute.provider_tree [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 601.354017] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg 918502add142482a9417022bf3831399 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 601.364174] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 918502add142482a9417022bf3831399 [ 601.369239] env[64020]: DEBUG nova.network.neutron [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 601.370883] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 6fedf3ddf5f94800a090924496a33cb6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 601.380674] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6fedf3ddf5f94800a090924496a33cb6 [ 601.397345] env[64020]: DEBUG nova.network.neutron [-] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 601.397345] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b1da9e5d9b1c4a3a8780b4cce68222cf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 601.404831] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1da9e5d9b1c4a3a8780b4cce68222cf [ 601.422161] env[64020]: INFO nova.compute.manager [-] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Took 1.04 seconds to deallocate network for instance. [ 601.424992] env[64020]: DEBUG nova.compute.claims [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 601.425133] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.431845] env[64020]: DEBUG nova.compute.manager [req-b16a3bdc-860f-4b5f-b9c6-c7b5ff51551d req-33d02e3f-1702-4171-84a3-387e612c190a service nova] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Received event network-changed-5820ea5a-bebc-46c4-a6a9-3551052bf734 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 601.432051] env[64020]: DEBUG nova.compute.manager [req-b16a3bdc-860f-4b5f-b9c6-c7b5ff51551d req-33d02e3f-1702-4171-84a3-387e612c190a service nova] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Refreshing instance network info cache due to event network-changed-5820ea5a-bebc-46c4-a6a9-3551052bf734. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 601.432289] env[64020]: DEBUG oslo_concurrency.lockutils [req-b16a3bdc-860f-4b5f-b9c6-c7b5ff51551d req-33d02e3f-1702-4171-84a3-387e612c190a service nova] Acquiring lock "refresh_cache-7529a478-7efe-4dff-8429-810549f56bd7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 601.432429] env[64020]: DEBUG oslo_concurrency.lockutils [req-b16a3bdc-860f-4b5f-b9c6-c7b5ff51551d req-33d02e3f-1702-4171-84a3-387e612c190a service nova] Acquired lock "refresh_cache-7529a478-7efe-4dff-8429-810549f56bd7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 601.432581] env[64020]: DEBUG nova.network.neutron [req-b16a3bdc-860f-4b5f-b9c6-c7b5ff51551d req-33d02e3f-1702-4171-84a3-387e612c190a service nova] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Refreshing network info cache for port 5820ea5a-bebc-46c4-a6a9-3551052bf734 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 601.433000] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-b16a3bdc-860f-4b5f-b9c6-c7b5ff51551d req-33d02e3f-1702-4171-84a3-387e612c190a service nova] Expecting reply to msg cfcc203edbba4397a044c58a75343381 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 601.440943] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfcc203edbba4397a044c58a75343381 [ 601.653243] env[64020]: DEBUG oslo_concurrency.lockutils [req-a40f7dc4-b401-477b-a0d3-62a8f7c0d3df req-7a86136b-02a9-497c-a910-96bb1171f0ae service nova] Releasing lock "refresh_cache-b18fca61-2b17-4758-824d-e02d2cb342d8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 601.653514] env[64020]: DEBUG nova.compute.manager [req-a40f7dc4-b401-477b-a0d3-62a8f7c0d3df req-7a86136b-02a9-497c-a910-96bb1171f0ae service nova] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Received event network-vif-deleted-3d243fb0-6d5c-4646-b594-0f1677cf790f {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 601.672356] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Acquiring lock "bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.672488] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Lock "bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 601.856880] env[64020]: DEBUG nova.scheduler.client.report [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 601.859288] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg c166390ffafa4c578fcc5676a129a885 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 601.872135] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Releasing lock "refresh_cache-9d4a7822-643e-4bdb-b431-a16e5d183100" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 601.872386] env[64020]: DEBUG nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 601.872543] env[64020]: DEBUG nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 601.872701] env[64020]: DEBUG nova.network.neutron [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 601.878335] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c166390ffafa4c578fcc5676a129a885 [ 601.901572] env[64020]: DEBUG nova.network.neutron [-] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 601.902110] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0b61cf0e09b644f4aceb4f15ea2f1f77 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 601.910283] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b61cf0e09b644f4aceb4f15ea2f1f77 [ 601.924382] env[64020]: DEBUG nova.network.neutron [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 601.925528] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 075c3e18fc154b61bc1d782c9a0d4cdf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 601.936362] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 075c3e18fc154b61bc1d782c9a0d4cdf [ 602.007425] env[64020]: DEBUG nova.network.neutron [req-b16a3bdc-860f-4b5f-b9c6-c7b5ff51551d req-33d02e3f-1702-4171-84a3-387e612c190a service nova] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 602.357714] env[64020]: DEBUG nova.network.neutron [req-b16a3bdc-860f-4b5f-b9c6-c7b5ff51551d req-33d02e3f-1702-4171-84a3-387e612c190a service nova] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 602.358230] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-b16a3bdc-860f-4b5f-b9c6-c7b5ff51551d req-33d02e3f-1702-4171-84a3-387e612c190a service nova] Expecting reply to msg 857586a7a80c4c6c8fd284394310caa4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 602.362980] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.920s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 602.362980] env[64020]: ERROR nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 32bd22fb-e5de-4222-8ff3-1328a2144ded, please check neutron logs for more information. [ 602.362980] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Traceback (most recent call last): [ 602.362980] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 602.362980] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] self.driver.spawn(context, instance, image_meta, [ 602.362980] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 602.362980] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 602.362980] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 602.362980] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] vm_ref = self.build_virtual_machine(instance, [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] vif_infos = vmwarevif.get_vif_info(self._session, [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] for vif in network_info: [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] return self._sync_wrapper(fn, *args, **kwargs) [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] self.wait() [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] self[:] = self._gt.wait() [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] return self._exit_event.wait() [ 602.363311] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] result = hub.switch() [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] return self.greenlet.switch() [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] result = function(*args, **kwargs) [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] return func(*args, **kwargs) [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] raise e [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] nwinfo = self.network_api.allocate_for_instance( [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 602.363686] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] created_port_ids = self._update_ports_for_instance( [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] with excutils.save_and_reraise_exception(): [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] self.force_reraise() [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] raise self.value [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] updated_port = self._update_port( [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] _ensure_no_port_binding_failure(port) [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 602.364079] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] raise exception.PortBindingFailed(port_id=port['id']) [ 602.364390] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] nova.exception.PortBindingFailed: Binding failed for port 32bd22fb-e5de-4222-8ff3-1328a2144ded, please check neutron logs for more information. [ 602.364390] env[64020]: ERROR nova.compute.manager [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] [ 602.364390] env[64020]: DEBUG nova.compute.utils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Binding failed for port 32bd22fb-e5de-4222-8ff3-1328a2144ded, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 602.364390] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 8.779s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 602.365558] env[64020]: INFO nova.compute.claims [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 602.367074] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 6080a201db094cbabe41873f35c8dabd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 602.368217] env[64020]: DEBUG nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Build of instance bbd58f57-9900-4d5d-847e-5275705e97f2 was re-scheduled: Binding failed for port 32bd22fb-e5de-4222-8ff3-1328a2144ded, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 602.368675] env[64020]: DEBUG nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 602.368879] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Acquiring lock "refresh_cache-bbd58f57-9900-4d5d-847e-5275705e97f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 602.369015] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Acquired lock "refresh_cache-bbd58f57-9900-4d5d-847e-5275705e97f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 602.369209] env[64020]: DEBUG nova.network.neutron [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 602.369524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg feb08ead9bc94f049e283391e186ed0f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 602.379702] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 857586a7a80c4c6c8fd284394310caa4 [ 602.385278] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg feb08ead9bc94f049e283391e186ed0f [ 602.404874] env[64020]: INFO nova.compute.manager [-] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Took 1.05 seconds to deallocate network for instance. [ 602.407970] env[64020]: DEBUG nova.compute.claims [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 602.407970] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 602.411891] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6080a201db094cbabe41873f35c8dabd [ 602.429169] env[64020]: DEBUG nova.network.neutron [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 602.432909] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 6685050a809f43279e8b50f2b82800a6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 602.442320] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6685050a809f43279e8b50f2b82800a6 [ 602.860599] env[64020]: DEBUG oslo_concurrency.lockutils [req-b16a3bdc-860f-4b5f-b9c6-c7b5ff51551d req-33d02e3f-1702-4171-84a3-387e612c190a service nova] Releasing lock "refresh_cache-7529a478-7efe-4dff-8429-810549f56bd7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 602.860953] env[64020]: DEBUG nova.compute.manager [req-b16a3bdc-860f-4b5f-b9c6-c7b5ff51551d req-33d02e3f-1702-4171-84a3-387e612c190a service nova] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Received event network-vif-deleted-5820ea5a-bebc-46c4-a6a9-3551052bf734 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 602.873169] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 3aee08d9bfbf44cdbf64ff91c3e68795 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 602.888154] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3aee08d9bfbf44cdbf64ff91c3e68795 [ 602.936347] env[64020]: DEBUG nova.network.neutron [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 602.936347] env[64020]: INFO nova.compute.manager [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] [instance: 9d4a7822-643e-4bdb-b431-a16e5d183100] Took 1.06 seconds to deallocate network for instance. [ 602.936347] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg 4d336202467545bc8f1beb30c5857986 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 603.000965] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d336202467545bc8f1beb30c5857986 [ 603.140424] env[64020]: DEBUG nova.network.neutron [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 603.140925] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg a2605f7ece854af889cfa7c00265f22a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 603.152782] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2605f7ece854af889cfa7c00265f22a [ 603.440443] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg e9e9b821451243a2a6d638c56984aa77 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 603.495897] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9e9b821451243a2a6d638c56984aa77 [ 603.595870] env[64020]: ERROR nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b92962a4-4c4a-4ade-9372-98c32103ec91, please check neutron logs for more information. [ 603.595870] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 603.595870] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 603.595870] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 603.595870] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 603.595870] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 603.595870] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 603.595870] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 603.595870] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 603.595870] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 603.595870] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 603.595870] env[64020]: ERROR nova.compute.manager raise self.value [ 603.595870] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 603.595870] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 603.595870] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 603.595870] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 603.596371] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 603.596371] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 603.596371] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b92962a4-4c4a-4ade-9372-98c32103ec91, please check neutron logs for more information. [ 603.596371] env[64020]: ERROR nova.compute.manager [ 603.596371] env[64020]: Traceback (most recent call last): [ 603.596371] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 603.596371] env[64020]: listener.cb(fileno) [ 603.596371] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 603.596371] env[64020]: result = function(*args, **kwargs) [ 603.596371] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 603.596371] env[64020]: return func(*args, **kwargs) [ 603.596371] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 603.596371] env[64020]: raise e [ 603.596371] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 603.596371] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 603.596371] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 603.596371] env[64020]: created_port_ids = self._update_ports_for_instance( [ 603.596371] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 603.596371] env[64020]: with excutils.save_and_reraise_exception(): [ 603.596371] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 603.596371] env[64020]: self.force_reraise() [ 603.596371] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 603.596371] env[64020]: raise self.value [ 603.596371] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 603.596371] env[64020]: updated_port = self._update_port( [ 603.596371] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 603.596371] env[64020]: _ensure_no_port_binding_failure(port) [ 603.596371] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 603.596371] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 603.597216] env[64020]: nova.exception.PortBindingFailed: Binding failed for port b92962a4-4c4a-4ade-9372-98c32103ec91, please check neutron logs for more information. [ 603.597216] env[64020]: Removing descriptor: 17 [ 603.597216] env[64020]: ERROR nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b92962a4-4c4a-4ade-9372-98c32103ec91, please check neutron logs for more information. [ 603.597216] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Traceback (most recent call last): [ 603.597216] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 603.597216] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] yield resources [ 603.597216] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 603.597216] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] self.driver.spawn(context, instance, image_meta, [ 603.597216] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 603.597216] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 603.597216] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 603.597216] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] vm_ref = self.build_virtual_machine(instance, [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] vif_infos = vmwarevif.get_vif_info(self._session, [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] for vif in network_info: [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] return self._sync_wrapper(fn, *args, **kwargs) [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] self.wait() [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] self[:] = self._gt.wait() [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] return self._exit_event.wait() [ 603.597568] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] result = hub.switch() [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] return self.greenlet.switch() [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] result = function(*args, **kwargs) [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] return func(*args, **kwargs) [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] raise e [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] nwinfo = self.network_api.allocate_for_instance( [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 603.597915] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] created_port_ids = self._update_ports_for_instance( [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] with excutils.save_and_reraise_exception(): [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] self.force_reraise() [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] raise self.value [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] updated_port = self._update_port( [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] _ensure_no_port_binding_failure(port) [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 603.598309] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] raise exception.PortBindingFailed(port_id=port['id']) [ 603.598637] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] nova.exception.PortBindingFailed: Binding failed for port b92962a4-4c4a-4ade-9372-98c32103ec91, please check neutron logs for more information. [ 603.598637] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] [ 603.598637] env[64020]: INFO nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Terminating instance [ 603.599788] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquiring lock "refresh_cache-5290c09b-d77c-4d27-b367-9de52f54d16a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 603.599942] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquired lock "refresh_cache-5290c09b-d77c-4d27-b367-9de52f54d16a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 603.600167] env[64020]: DEBUG nova.network.neutron [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 603.603680] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 2d2638ec39d4448487859dedc3a4ed09 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 603.610808] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d2638ec39d4448487859dedc3a4ed09 [ 603.643013] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Releasing lock "refresh_cache-bbd58f57-9900-4d5d-847e-5275705e97f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 603.643234] env[64020]: DEBUG nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 603.643415] env[64020]: DEBUG nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 603.643575] env[64020]: DEBUG nova.network.neutron [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 603.683669] env[64020]: DEBUG nova.network.neutron [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 603.684256] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg eb36c36130be441b8a31ef092eb2e3ef in queue reply_57893177120949e6a93cb88e15cd42b4 [ 603.695087] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb36c36130be441b8a31ef092eb2e3ef [ 603.703808] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2baa48c9-5a4e-49c8-9dab-e4b4f6714214 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.711908] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e25f06c2-ffec-448a-9870-bde7ca54b21a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.751477] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69b0005e-b1bd-44d3-ae83-b4926d5fc384 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.759640] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00e0b779-4543-40cd-b9c5-c841b79e82d9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.775684] env[64020]: DEBUG nova.compute.provider_tree [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 603.776220] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg de666cba50d344339b7909fbc7e88afa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 603.783771] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de666cba50d344339b7909fbc7e88afa [ 603.979506] env[64020]: INFO nova.scheduler.client.report [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Deleted allocations for instance 9d4a7822-643e-4bdb-b431-a16e5d183100 [ 603.996334] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Expecting reply to msg e3f0f6a638944a8780b305a7f6c524b9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 604.017406] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Acquiring lock "618c3b06-1fd8-45d3-9c59-61c0e202a299" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 604.017644] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Lock "618c3b06-1fd8-45d3-9c59-61c0e202a299" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 604.023066] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3f0f6a638944a8780b305a7f6c524b9 [ 604.186392] env[64020]: DEBUG nova.network.neutron [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 604.186979] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg 06d280af6b1040698f9bff726f6f6a6b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 604.200245] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06d280af6b1040698f9bff726f6f6a6b [ 604.279412] env[64020]: DEBUG nova.scheduler.client.report [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 604.281917] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 0e17f5ae1a4a4452af9898aedb75764f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 604.292738] env[64020]: DEBUG nova.network.neutron [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 604.296934] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e17f5ae1a4a4452af9898aedb75764f [ 604.498502] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7628b92c-bfa5-4382-a86c-c4ae3de6c1f9 tempest-ServerDiagnosticsTest-2035304071 tempest-ServerDiagnosticsTest-2035304071-project-member] Lock "9d4a7822-643e-4bdb-b431-a16e5d183100" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 24.891s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 604.499140] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 6b6a0df78e4645d4be6cd2cc6d7670e6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 604.511261] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6b6a0df78e4645d4be6cd2cc6d7670e6 [ 604.525673] env[64020]: DEBUG nova.network.neutron [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 604.526189] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg af128cfc829240d58f3fa4fb4caa71c2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 604.535550] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af128cfc829240d58f3fa4fb4caa71c2 [ 604.689901] env[64020]: INFO nova.compute.manager [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] [instance: bbd58f57-9900-4d5d-847e-5275705e97f2] Took 1.05 seconds to deallocate network for instance. [ 604.692854] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg 6e93d828d3cd4be2900ab54dadb64944 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 604.754821] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6e93d828d3cd4be2900ab54dadb64944 [ 604.787333] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.421s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 604.787333] env[64020]: DEBUG nova.compute.manager [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 604.787333] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 80f7fdfc49564f15b33b4486c7811363 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 604.787703] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 10.187s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 604.791323] env[64020]: INFO nova.compute.claims [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 604.800812] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 6564db8a46a64b5499dd44004cb4c0f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 604.836646] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80f7fdfc49564f15b33b4486c7811363 [ 604.851549] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6564db8a46a64b5499dd44004cb4c0f5 [ 605.001742] env[64020]: DEBUG nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 605.003461] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 561e7da37f07481192bda1b21dfa0281 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 605.028577] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Releasing lock "refresh_cache-5290c09b-d77c-4d27-b367-9de52f54d16a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 605.028931] env[64020]: DEBUG nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 605.029271] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 605.030254] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5e40c42a-60c4-4ed7-bd9e-d7b193616324 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.039192] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-853d8659-1ba4-4d8d-9dd0-d2969af28190 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.053232] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 561e7da37f07481192bda1b21dfa0281 [ 605.073349] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5290c09b-d77c-4d27-b367-9de52f54d16a could not be found. [ 605.073599] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 605.073787] env[64020]: INFO nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Took 0.04 seconds to destroy the instance on the hypervisor. [ 605.074033] env[64020]: DEBUG oslo.service.loopingcall [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 605.074235] env[64020]: DEBUG nova.compute.manager [-] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 605.074329] env[64020]: DEBUG nova.network.neutron [-] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 605.121871] env[64020]: DEBUG nova.network.neutron [-] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 605.122190] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 355f44e5173f480caedd4f5f34a1c1ba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 605.131993] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 355f44e5173f480caedd4f5f34a1c1ba [ 605.197720] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg 46b0f428518c446383c0ebc120e15b69 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 605.245662] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46b0f428518c446383c0ebc120e15b69 [ 605.295693] env[64020]: DEBUG nova.compute.utils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 605.296359] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 5ee5477217194ec98627ecf4b33d9184 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 605.298354] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg eafdab11209f45cabef7c4321e5ca584 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 605.302590] env[64020]: DEBUG nova.compute.manager [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Not allocating networking since 'none' was specified. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 605.305971] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ee5477217194ec98627ecf4b33d9184 [ 605.306508] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eafdab11209f45cabef7c4321e5ca584 [ 605.526640] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 605.558366] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Acquiring lock "4478746a-11a6-480d-84c1-c6f04ab33505" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 605.558588] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Lock "4478746a-11a6-480d-84c1-c6f04ab33505" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 605.626282] env[64020]: DEBUG nova.network.neutron [-] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 605.626755] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a29cef4952234c9783570a16d77223fc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 605.638199] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a29cef4952234c9783570a16d77223fc [ 605.661547] env[64020]: DEBUG nova.compute.manager [req-6adf91a6-8c5b-487f-b3e8-40e95999c211 req-19bb210a-03b2-493c-95e1-3ed1c0b12ddc service nova] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Received event network-changed-b92962a4-4c4a-4ade-9372-98c32103ec91 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 605.661732] env[64020]: DEBUG nova.compute.manager [req-6adf91a6-8c5b-487f-b3e8-40e95999c211 req-19bb210a-03b2-493c-95e1-3ed1c0b12ddc service nova] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Refreshing instance network info cache due to event network-changed-b92962a4-4c4a-4ade-9372-98c32103ec91. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 605.661938] env[64020]: DEBUG oslo_concurrency.lockutils [req-6adf91a6-8c5b-487f-b3e8-40e95999c211 req-19bb210a-03b2-493c-95e1-3ed1c0b12ddc service nova] Acquiring lock "refresh_cache-5290c09b-d77c-4d27-b367-9de52f54d16a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 605.662074] env[64020]: DEBUG oslo_concurrency.lockutils [req-6adf91a6-8c5b-487f-b3e8-40e95999c211 req-19bb210a-03b2-493c-95e1-3ed1c0b12ddc service nova] Acquired lock "refresh_cache-5290c09b-d77c-4d27-b367-9de52f54d16a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 605.662228] env[64020]: DEBUG nova.network.neutron [req-6adf91a6-8c5b-487f-b3e8-40e95999c211 req-19bb210a-03b2-493c-95e1-3ed1c0b12ddc service nova] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Refreshing network info cache for port b92962a4-4c4a-4ade-9372-98c32103ec91 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 605.662653] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-6adf91a6-8c5b-487f-b3e8-40e95999c211 req-19bb210a-03b2-493c-95e1-3ed1c0b12ddc service nova] Expecting reply to msg 898a95f5bd9a472f91cb9e72802ab514 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 605.672294] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 898a95f5bd9a472f91cb9e72802ab514 [ 605.721038] env[64020]: INFO nova.scheduler.client.report [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Deleted allocations for instance bbd58f57-9900-4d5d-847e-5275705e97f2 [ 605.729459] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Expecting reply to msg 6ee5e123f9834e49bdabf9f8523412e3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 605.742381] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ee5e123f9834e49bdabf9f8523412e3 [ 605.806021] env[64020]: DEBUG nova.compute.manager [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 605.806021] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg e275006eb2254859a495dd6ba6b84449 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 605.890967] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e275006eb2254859a495dd6ba6b84449 [ 606.108118] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3781bbe4-0ef1-40d0-9033-49a2187eb39f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.111753] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f001c5a-42e2-4155-a38e-b231e6fd329c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.152382] env[64020]: INFO nova.compute.manager [-] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Took 1.08 seconds to deallocate network for instance. [ 606.155729] env[64020]: DEBUG nova.compute.claims [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 606.155860] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 606.156654] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c4d11f0-2556-4283-ae54-6e039518f1ba {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.169053] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-830c56dc-fb58-44b0-a9a1-21d74671281b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.193374] env[64020]: DEBUG nova.compute.provider_tree [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 606.193968] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg acf2fd370b3e4ab089d38682a28edbfb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 606.195956] env[64020]: DEBUG nova.network.neutron [req-6adf91a6-8c5b-487f-b3e8-40e95999c211 req-19bb210a-03b2-493c-95e1-3ed1c0b12ddc service nova] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 606.204652] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg acf2fd370b3e4ab089d38682a28edbfb [ 606.232064] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0904b1f6-dedc-4ed6-acdb-b0ac45aa62b7 tempest-ServersTestJSON-1988027487 tempest-ServersTestJSON-1988027487-project-member] Lock "bbd58f57-9900-4d5d-847e-5275705e97f2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 24.725s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 606.232899] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 9331a7808356456fbda4c913dd192827 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 606.241403] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Acquiring lock "9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 606.241630] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Lock "9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 606.246328] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9331a7808356456fbda4c913dd192827 [ 606.309378] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 03dab6a63eb34233901fcce46c468a4a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 606.346523] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03dab6a63eb34233901fcce46c468a4a [ 606.347684] env[64020]: DEBUG nova.network.neutron [req-6adf91a6-8c5b-487f-b3e8-40e95999c211 req-19bb210a-03b2-493c-95e1-3ed1c0b12ddc service nova] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 606.348219] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-6adf91a6-8c5b-487f-b3e8-40e95999c211 req-19bb210a-03b2-493c-95e1-3ed1c0b12ddc service nova] Expecting reply to msg 0bdb282a504b4d6bb247bb995981115a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 606.356058] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0bdb282a504b4d6bb247bb995981115a [ 606.553927] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Acquiring lock "cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 606.554221] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Lock "cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 606.699486] env[64020]: DEBUG nova.scheduler.client.report [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 606.702003] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 1df8d48f643d4108baf896aa757e53c4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 606.731857] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1df8d48f643d4108baf896aa757e53c4 [ 606.737333] env[64020]: DEBUG nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 606.738745] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 3bc09b98f94b4c7496c31a4610e0313b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 606.780100] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3bc09b98f94b4c7496c31a4610e0313b [ 606.822237] env[64020]: DEBUG nova.compute.manager [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 606.847779] env[64020]: DEBUG nova.virt.hardware [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 606.848110] env[64020]: DEBUG nova.virt.hardware [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 606.848290] env[64020]: DEBUG nova.virt.hardware [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 606.848510] env[64020]: DEBUG nova.virt.hardware [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 606.848686] env[64020]: DEBUG nova.virt.hardware [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 606.848849] env[64020]: DEBUG nova.virt.hardware [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 606.849081] env[64020]: DEBUG nova.virt.hardware [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 606.849255] env[64020]: DEBUG nova.virt.hardware [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 606.849630] env[64020]: DEBUG nova.virt.hardware [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 606.849676] env[64020]: DEBUG nova.virt.hardware [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 606.849841] env[64020]: DEBUG nova.virt.hardware [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 606.850723] env[64020]: DEBUG oslo_concurrency.lockutils [req-6adf91a6-8c5b-487f-b3e8-40e95999c211 req-19bb210a-03b2-493c-95e1-3ed1c0b12ddc service nova] Releasing lock "refresh_cache-5290c09b-d77c-4d27-b367-9de52f54d16a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 606.851776] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3a7bdd94-9fcc-48db-99fd-51910bbb8b6e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.860090] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-601bb202-bd35-414c-ba74-c1d936c1b316 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.874197] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 606.886051] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Creating folder: OpenStack. Parent ref: group-v4. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 606.886051] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-e988a754-7028-416d-826d-708585f13bd1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.900184] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Created folder: OpenStack in parent group-v4. [ 606.900532] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Creating folder: Project (b0448cdffe2840229a141c081e4de7de). Parent ref: group-v110249. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 606.901019] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-089e090c-06f3-4882-9a7f-fe417dc5fbf9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.911092] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Created folder: Project (b0448cdffe2840229a141c081e4de7de) in parent group-v110249. [ 606.912261] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Creating folder: Instances. Parent ref: group-v110250. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 606.912554] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-aa6c7d76-4451-4380-8b60-17ee6d058b64 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.926423] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Created folder: Instances in parent group-v110250. [ 606.926948] env[64020]: DEBUG oslo.service.loopingcall [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 606.927315] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 606.927708] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-4d90e65b-6555-46ba-86d7-b1b0341814b7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.948750] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 606.948750] env[64020]: value = "task-407816" [ 606.948750] env[64020]: _type = "Task" [ 606.948750] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 606.964381] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407816, 'name': CreateVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 607.212430] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.421s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 607.212430] env[64020]: DEBUG nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 607.212430] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 1e9727387546410a8ae4619896243689 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 607.215524] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.390s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 607.215524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg 72def1b3c32c4019a209862f38a7149c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 607.256761] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e9727387546410a8ae4619896243689 [ 607.265593] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 607.289640] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72def1b3c32c4019a209862f38a7149c [ 607.460101] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407816, 'name': CreateVM_Task, 'duration_secs': 0.424167} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 607.460532] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 607.461627] env[64020]: DEBUG oslo_vmware.service [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce53ff1b-ff1f-43f0-b3fa-15f4df2969f9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.467663] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 607.467926] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 607.470789] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 607.470789] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8fb83885-de3c-4565-94f9-9c438e2ddde3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.479865] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 607.479865] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5248eb67-43b6-1dab-0712-e7372b03cfb4" [ 607.479865] env[64020]: _type = "Task" [ 607.479865] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 607.485801] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5248eb67-43b6-1dab-0712-e7372b03cfb4, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 607.732227] env[64020]: DEBUG nova.compute.utils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 607.733044] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg a930b2803df64638be86cb811b29767d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 607.739062] env[64020]: DEBUG nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 607.739062] env[64020]: DEBUG nova.network.neutron [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 607.747828] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a930b2803df64638be86cb811b29767d [ 607.974492] env[64020]: DEBUG nova.policy [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '519f1efd58d14fd5bf049f8adda3e65b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd70af9cb753242408c408c99ffaacab8', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 607.999100] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 607.999362] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 607.999589] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 607.999729] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 608.000181] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 608.000593] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-02f2667f-6fcd-4d2d-bffd-9cb743c5a542 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.018532] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 608.018789] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 608.019722] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c85f59b-6877-458a-bdbd-76e70adc9e8d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.035459] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-ea1e2eb6-fb87-478d-8c27-5ec8bbc553a9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.044404] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 608.044404] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52ad5b05-d142-ba00-93a9-014a3d0ef1c5" [ 608.044404] env[64020]: _type = "Task" [ 608.044404] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 608.053936] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52ad5b05-d142-ba00-93a9-014a3d0ef1c5, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 608.108693] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e082294d-b20f-4329-b49d-25c6234c4245 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.117008] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9be42588-9486-40ba-aade-6a265bfd5ea3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.153525] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afa8b2f9-b10b-4902-ac1d-e424e856e372 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.161291] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4aa0e0e4-8960-4c54-8962-9d7bec4ae4bd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.175100] env[64020]: DEBUG nova.compute.provider_tree [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 608.175647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg 4da4f27a86204157b1879b35d31f0e8c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 608.183971] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4da4f27a86204157b1879b35d31f0e8c [ 608.237931] env[64020]: DEBUG nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 608.239791] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 2f13464f485b44ba9d46dcda23b75ef3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 608.277720] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f13464f485b44ba9d46dcda23b75ef3 [ 608.554419] env[64020]: DEBUG nova.compute.manager [req-87ade307-eb68-46a4-a4f2-532422f704df req-ff46ca9c-cf37-47a6-9702-2c046a4b9b3c service nova] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Received event network-vif-deleted-b92962a4-4c4a-4ade-9372-98c32103ec91 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 608.560442] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Preparing fetch location {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 608.560815] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Creating directory with path [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 608.561089] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-d83ebf9e-19fd-4322-b70f-e2e5c74c8b92 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.594628] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Created directory with path [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 608.594829] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Fetch image to [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 608.594998] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Downloading image file data 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 to [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk on the data store datastore1 {{(pid=64020) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 608.596138] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e58f59a-8bc3-4f9a-9f2f-ef7442af1a7f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.604690] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7893075d-2a61-4e2d-99ce-c4e98159e530 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.618377] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a817d95-4f44-418d-832c-7a519b5160b8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.656996] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-516a1517-a23a-4616-9b9d-36bc49ceb36a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.672765] env[64020]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-bab71336-93a7-44f8-adf8-6f9bb8f2a687 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.678672] env[64020]: DEBUG nova.scheduler.client.report [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 608.681040] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg b3b0f5e229694cf8a3d5d340a3490777 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 608.696417] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3b0f5e229694cf8a3d5d340a3490777 [ 608.745357] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 8864d0bf8eb145cfb111fb06454a0d1b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 608.769408] env[64020]: DEBUG nova.virt.vmwareapi.images [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Downloading image file data 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 to the data store datastore1 {{(pid=64020) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 608.790821] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8864d0bf8eb145cfb111fb06454a0d1b [ 608.833049] env[64020]: DEBUG oslo_vmware.rw_handles [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=64020) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 609.192275] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.970s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 609.192275] env[64020]: ERROR nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 06ebe7b0-7b82-4f26-85b9-cc69537dc66f, please check neutron logs for more information. [ 609.192275] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Traceback (most recent call last): [ 609.192275] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 609.192275] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] self.driver.spawn(context, instance, image_meta, [ 609.192275] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 609.192275] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 609.192275] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 609.192275] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] vm_ref = self.build_virtual_machine(instance, [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] vif_infos = vmwarevif.get_vif_info(self._session, [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] for vif in network_info: [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] return self._sync_wrapper(fn, *args, **kwargs) [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] self.wait() [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] self[:] = self._gt.wait() [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] return self._exit_event.wait() [ 609.192726] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] result = hub.switch() [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] return self.greenlet.switch() [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] result = function(*args, **kwargs) [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] return func(*args, **kwargs) [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] raise e [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] nwinfo = self.network_api.allocate_for_instance( [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 609.193206] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] created_port_ids = self._update_ports_for_instance( [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] with excutils.save_and_reraise_exception(): [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] self.force_reraise() [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] raise self.value [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] updated_port = self._update_port( [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] _ensure_no_port_binding_failure(port) [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 609.193531] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] raise exception.PortBindingFailed(port_id=port['id']) [ 609.193831] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] nova.exception.PortBindingFailed: Binding failed for port 06ebe7b0-7b82-4f26-85b9-cc69537dc66f, please check neutron logs for more information. [ 609.193831] env[64020]: ERROR nova.compute.manager [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] [ 609.193831] env[64020]: DEBUG nova.compute.utils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Binding failed for port 06ebe7b0-7b82-4f26-85b9-cc69537dc66f, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 609.193831] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.635s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 609.193831] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 92dcb5b581fc4a14af57dea45b5cb6c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 609.193978] env[64020]: DEBUG nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Build of instance c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c was re-scheduled: Binding failed for port 06ebe7b0-7b82-4f26-85b9-cc69537dc66f, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 609.193978] env[64020]: DEBUG nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 609.193978] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Acquiring lock "refresh_cache-c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 609.193978] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Acquired lock "refresh_cache-c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 609.194115] env[64020]: DEBUG nova.network.neutron [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 609.194115] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg dd594b817255428d8c95119056c2c6b8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 609.208356] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dd594b817255428d8c95119056c2c6b8 [ 609.248268] env[64020]: DEBUG nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 609.251062] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92dcb5b581fc4a14af57dea45b5cb6c8 [ 609.281288] env[64020]: DEBUG nova.virt.hardware [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 609.281555] env[64020]: DEBUG nova.virt.hardware [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 609.281745] env[64020]: DEBUG nova.virt.hardware [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 609.281889] env[64020]: DEBUG nova.virt.hardware [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 609.282042] env[64020]: DEBUG nova.virt.hardware [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 609.282277] env[64020]: DEBUG nova.virt.hardware [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 609.282439] env[64020]: DEBUG nova.virt.hardware [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 609.282589] env[64020]: DEBUG nova.virt.hardware [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 609.282827] env[64020]: DEBUG nova.virt.hardware [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 609.283020] env[64020]: DEBUG nova.virt.hardware [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 609.283202] env[64020]: DEBUG nova.virt.hardware [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 609.284852] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb3c8555-093f-447f-a63b-472732f75c3e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.304798] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7ff7b07-ec9f-468a-9593-8cdc97cb5b4e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.595037] env[64020]: DEBUG oslo_vmware.rw_handles [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Completed reading data from the image iterator. {{(pid=64020) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 609.595276] env[64020]: DEBUG oslo_vmware.rw_handles [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=64020) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 609.740662] env[64020]: DEBUG nova.virt.vmwareapi.images [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Downloaded image file data 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 to vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk on the data store datastore1 {{(pid=64020) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 609.742754] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Caching image {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 609.743021] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Copying Virtual Disk [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk to [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 609.743304] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-3dd9afbd-d30e-46d7-90dd-677d322e33be {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.757626] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 609.757626] env[64020]: value = "task-407817" [ 609.757626] env[64020]: _type = "Task" [ 609.757626] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 609.769468] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407817, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 609.834474] env[64020]: DEBUG nova.network.neutron [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 609.861786] env[64020]: DEBUG nova.network.neutron [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Successfully created port: 57a9791c-3911-480c-bb77-c7950afb78c3 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 610.050027] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f63d4913-c118-497f-b064-dfdbf05cf8cf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.057975] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5c5960a-4520-4d4c-8475-cb034130b199 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.102555] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-beedc4e2-f9b6-4c03-9aaa-1d86ee8402f8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.110991] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1492aee6-b11e-467b-9321-3f95b48f233b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.124913] env[64020]: DEBUG nova.compute.provider_tree [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 610.125255] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg bfb071a40ebf4c0e8d015e17cfbf8e73 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 610.134338] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bfb071a40ebf4c0e8d015e17cfbf8e73 [ 610.274757] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407817, 'name': CopyVirtualDisk_Task} progress is 100%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 610.574781] env[64020]: DEBUG nova.network.neutron [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 610.574781] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg 5748b0da000748bebd92504d4b87527c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 610.581936] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5748b0da000748bebd92504d4b87527c [ 610.628793] env[64020]: DEBUG nova.scheduler.client.report [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 610.637554] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg f1334058f387468e8cdfeafaebe56ee1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 610.654640] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f1334058f387468e8cdfeafaebe56ee1 [ 610.769042] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407817, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.665635} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 610.769302] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Copied Virtual Disk [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk to [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 610.769473] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleting the datastore file [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 610.772834] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-9a956ac4-5fbd-4adf-a393-eb9e8c8cfada {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.779749] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 610.779749] env[64020]: value = "task-407818" [ 610.779749] env[64020]: _type = "Task" [ 610.779749] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 610.793978] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407818, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 611.079207] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Releasing lock "refresh_cache-c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 611.079207] env[64020]: DEBUG nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 611.079207] env[64020]: DEBUG nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 611.079207] env[64020]: DEBUG nova.network.neutron [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 611.142780] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.955s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 611.143448] env[64020]: ERROR nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4c2e20e6-3e5d-4866-b20c-469a896012eb, please check neutron logs for more information. [ 611.143448] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Traceback (most recent call last): [ 611.143448] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 611.143448] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] self.driver.spawn(context, instance, image_meta, [ 611.143448] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 611.143448] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 611.143448] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 611.143448] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] vm_ref = self.build_virtual_machine(instance, [ 611.143448] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 611.143448] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] vif_infos = vmwarevif.get_vif_info(self._session, [ 611.143448] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] for vif in network_info: [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] return self._sync_wrapper(fn, *args, **kwargs) [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] self.wait() [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] self[:] = self._gt.wait() [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] return self._exit_event.wait() [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] result = hub.switch() [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 611.143896] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] return self.greenlet.switch() [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] result = function(*args, **kwargs) [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] return func(*args, **kwargs) [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] raise e [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] nwinfo = self.network_api.allocate_for_instance( [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] created_port_ids = self._update_ports_for_instance( [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] with excutils.save_and_reraise_exception(): [ 611.144254] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 611.144574] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] self.force_reraise() [ 611.144574] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 611.144574] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] raise self.value [ 611.144574] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 611.144574] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] updated_port = self._update_port( [ 611.144574] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 611.144574] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] _ensure_no_port_binding_failure(port) [ 611.144574] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 611.144574] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] raise exception.PortBindingFailed(port_id=port['id']) [ 611.144574] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] nova.exception.PortBindingFailed: Binding failed for port 4c2e20e6-3e5d-4866-b20c-469a896012eb, please check neutron logs for more information. [ 611.144574] env[64020]: ERROR nova.compute.manager [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] [ 611.144854] env[64020]: DEBUG nova.compute.utils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Binding failed for port 4c2e20e6-3e5d-4866-b20c-469a896012eb, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 611.145426] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.745s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 611.147139] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg b4a38f55d2de4f3e838614f2db24a714 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 611.153251] env[64020]: DEBUG nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Build of instance 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2 was re-scheduled: Binding failed for port 4c2e20e6-3e5d-4866-b20c-469a896012eb, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 611.153251] env[64020]: DEBUG nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 611.153251] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquiring lock "refresh_cache-4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 611.153598] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquired lock "refresh_cache-4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 611.153598] env[64020]: DEBUG nova.network.neutron [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 611.153917] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg cd3acac634864297ba86d07403519646 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 611.161741] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd3acac634864297ba86d07403519646 [ 611.204375] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b4a38f55d2de4f3e838614f2db24a714 [ 611.290002] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407818, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.022917} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 611.290318] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 611.290477] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Moving file from [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 to [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234. {{(pid=64020) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:334}} [ 611.290720] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MoveDatastoreFile_Task with opID=oslo.vmware-a526967b-3f63-442b-b5f1-940215654de7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.297726] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 611.297726] env[64020]: value = "task-407819" [ 611.297726] env[64020]: _type = "Task" [ 611.297726] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 611.307629] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407819, 'name': MoveDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 611.321101] env[64020]: DEBUG nova.network.neutron [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 611.321101] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg d85b633732184ae7ae2c1ee30641e310 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 611.329176] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d85b633732184ae7ae2c1ee30641e310 [ 611.728354] env[64020]: DEBUG nova.network.neutron [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 611.811543] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407819, 'name': MoveDatastoreFile_Task, 'duration_secs': 0.024227} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 611.814328] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] File moved {{(pid=64020) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:346}} [ 611.814578] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Cleaning up location [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:640}} [ 611.814690] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleting the datastore file [datastore1] vmware_temp/876db23f-55d9-43ee-9c28-9bec794652fb {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 611.815371] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-dd294088-1b11-4100-bdda-5ea1f5160b77 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.823223] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 611.823223] env[64020]: value = "task-407820" [ 611.823223] env[64020]: _type = "Task" [ 611.823223] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 611.824263] env[64020]: DEBUG nova.network.neutron [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 611.824966] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg e974e8ccd635486e8a00bd9a3ce3497e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 611.839232] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407820, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 611.844475] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e974e8ccd635486e8a00bd9a3ce3497e [ 611.991834] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1bcbcbc8-841c-411f-ab62-5fd0b365123e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.000456] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09ceff37-8eb7-4810-91fb-31d1434d629e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.033225] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-983d3b6a-f80d-4644-82df-f72609bec9ce {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.040650] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e1f711f-b9eb-46a6-81e7-635f26452af6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.054622] env[64020]: DEBUG nova.compute.provider_tree [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 612.054932] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 280e4d8c7b6e409dbd28ebd7cd5e5839 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 612.068210] env[64020]: DEBUG nova.network.neutron [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 612.068479] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 7c061706d30c4bdab20cb10684e5237d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 612.075880] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 280e4d8c7b6e409dbd28ebd7cd5e5839 [ 612.091636] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c061706d30c4bdab20cb10684e5237d [ 612.334039] env[64020]: INFO nova.compute.manager [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] [instance: c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c] Took 1.26 seconds to deallocate network for instance. [ 612.335784] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg 9c39f677dbc941eea63fcd9764a1885b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 612.336804] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407820, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.02602} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 612.337158] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 612.338449] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-e1242fce-159a-44b8-aede-1b26df881901 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.344304] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 612.344304] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52658acf-438e-d439-a53b-2de7d9926c9a" [ 612.344304] env[64020]: _type = "Task" [ 612.344304] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 612.352159] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52658acf-438e-d439-a53b-2de7d9926c9a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 612.388480] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c39f677dbc941eea63fcd9764a1885b [ 612.560175] env[64020]: DEBUG nova.scheduler.client.report [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 612.562835] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg fd40b3a12ec7486e8f5261212046d235 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 612.570215] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Releasing lock "refresh_cache-4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 612.570446] env[64020]: DEBUG nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 612.570612] env[64020]: DEBUG nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 612.570773] env[64020]: DEBUG nova.network.neutron [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 612.581265] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd40b3a12ec7486e8f5261212046d235 [ 612.605915] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Acquiring lock "5ac25e1d-007a-4eaa-811c-655b54efd7ec" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 612.606184] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Lock "5ac25e1d-007a-4eaa-811c-655b54efd7ec" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 612.622380] env[64020]: DEBUG nova.network.neutron [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 612.623007] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 3c4593cafaa54af181d1eb6245ffca52 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 612.630663] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c4593cafaa54af181d1eb6245ffca52 [ 612.842389] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg 56104c5673f74c98bc4b3d531f42f418 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 612.855342] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52658acf-438e-d439-a53b-2de7d9926c9a, 'name': SearchDatastore_Task, 'duration_secs': 0.020572} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 612.855659] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 612.855936] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 612.856244] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-4d5fc972-3d1d-4be7-afb9-9008275c7d2d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.863575] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 612.863575] env[64020]: value = "task-407821" [ 612.863575] env[64020]: _type = "Task" [ 612.863575] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 612.872495] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407821, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 612.891411] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 56104c5673f74c98bc4b3d531f42f418 [ 613.068147] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.922s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.068533] env[64020]: ERROR nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3a25c429-30d6-43fb-927d-555318c582a6, please check neutron logs for more information. [ 613.068533] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Traceback (most recent call last): [ 613.068533] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 613.068533] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] self.driver.spawn(context, instance, image_meta, [ 613.068533] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 613.068533] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 613.068533] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 613.068533] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] vm_ref = self.build_virtual_machine(instance, [ 613.068533] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 613.068533] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] vif_infos = vmwarevif.get_vif_info(self._session, [ 613.068533] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] for vif in network_info: [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] return self._sync_wrapper(fn, *args, **kwargs) [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] self.wait() [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] self[:] = self._gt.wait() [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] return self._exit_event.wait() [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] result = hub.switch() [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 613.068866] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] return self.greenlet.switch() [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] result = function(*args, **kwargs) [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] return func(*args, **kwargs) [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] raise e [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] nwinfo = self.network_api.allocate_for_instance( [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] created_port_ids = self._update_ports_for_instance( [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] with excutils.save_and_reraise_exception(): [ 613.069297] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 613.069627] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] self.force_reraise() [ 613.069627] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 613.069627] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] raise self.value [ 613.069627] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 613.069627] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] updated_port = self._update_port( [ 613.069627] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 613.069627] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] _ensure_no_port_binding_failure(port) [ 613.069627] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 613.069627] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] raise exception.PortBindingFailed(port_id=port['id']) [ 613.069627] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] nova.exception.PortBindingFailed: Binding failed for port 3a25c429-30d6-43fb-927d-555318c582a6, please check neutron logs for more information. [ 613.069627] env[64020]: ERROR nova.compute.manager [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] [ 613.069899] env[64020]: DEBUG nova.compute.utils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Binding failed for port 3a25c429-30d6-43fb-927d-555318c582a6, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 613.070471] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 14.383s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.071397] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg f3c79b42d8a5410cbd181055e7c01853 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 613.075506] env[64020]: DEBUG nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Build of instance 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3 was re-scheduled: Binding failed for port 3a25c429-30d6-43fb-927d-555318c582a6, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 613.075920] env[64020]: DEBUG nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 613.076536] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "refresh_cache-60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 613.076723] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquired lock "refresh_cache-60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 613.076888] env[64020]: DEBUG nova.network.neutron [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 613.077665] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg cb092865f7254a209e53006596755268 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 613.085087] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cb092865f7254a209e53006596755268 [ 613.122997] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3c79b42d8a5410cbd181055e7c01853 [ 613.126471] env[64020]: DEBUG nova.network.neutron [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 613.129356] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg cf00691e77e44fbf9f0f70f6b730bc9f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 613.139585] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cf00691e77e44fbf9f0f70f6b730bc9f [ 613.370940] env[64020]: INFO nova.scheduler.client.report [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Deleted allocations for instance c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c [ 613.380013] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407821, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.507285} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 613.380727] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Expecting reply to msg 11e66565c24f4a94b0e6976bf53aeaa5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 613.381385] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 613.382001] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 613.382001] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-37544533-601b-4b51-b187-a4c0b396e00d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.394952] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 613.394952] env[64020]: value = "task-407822" [ 613.394952] env[64020]: _type = "Task" [ 613.394952] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 613.398435] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 11e66565c24f4a94b0e6976bf53aeaa5 [ 613.404020] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407822, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 613.587102] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 4f523baa6fa446a0901f8cc7fc608946 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 613.607524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f523baa6fa446a0901f8cc7fc608946 [ 613.610790] env[64020]: DEBUG nova.network.neutron [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 613.631691] env[64020]: INFO nova.compute.manager [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2] Took 1.06 seconds to deallocate network for instance. [ 613.633490] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg d4c9fd2468dc4fd3ae1898c456694268 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 613.676814] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d4c9fd2468dc4fd3ae1898c456694268 [ 613.737122] env[64020]: DEBUG nova.network.neutron [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 613.737671] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 2bb484dc2629411a8ee4919d36abbc10 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 613.746072] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2bb484dc2629411a8ee4919d36abbc10 [ 613.883018] env[64020]: DEBUG oslo_concurrency.lockutils [None req-26a6d7a3-2f3f-4b00-8585-db57c00e435e tempest-ImagesOneServerTestJSON-1655624413 tempest-ImagesOneServerTestJSON-1655624413-project-member] Lock "c2d31dea-e4f9-48ee-9352-b5ddd1e8db6c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 31.410s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.883623] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 9e397f4660844106b23efbb880b1ca0b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 613.899048] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e397f4660844106b23efbb880b1ca0b [ 613.905464] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407822, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.067689} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 613.905734] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 613.906671] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b289060-2ae0-420f-9ede-aeae8f85b26b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.933772] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Reconfiguring VM instance instance-00000009 to attach disk [datastore1] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 613.933772] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-b8ecd188-bcce-4359-bfdf-31e20690119b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.966026] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 613.966026] env[64020]: value = "task-407823" [ 613.966026] env[64020]: _type = "Task" [ 613.966026] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 613.976722] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407823, 'name': ReconfigVM_Task} progress is 14%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 614.109907] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg a41569342bb649fe9aeb34ba7290e0c4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 614.138095] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 94c9a14f957045edad04839f9dd34e5e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 614.145623] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a41569342bb649fe9aeb34ba7290e0c4 [ 614.177306] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94c9a14f957045edad04839f9dd34e5e [ 614.240812] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Releasing lock "refresh_cache-60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 614.241089] env[64020]: DEBUG nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 614.241278] env[64020]: DEBUG nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 614.241446] env[64020]: DEBUG nova.network.neutron [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 614.279096] env[64020]: DEBUG nova.network.neutron [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 614.279788] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 7cfebb3b491944c98c3d50b34eac9fa4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 614.289721] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7cfebb3b491944c98c3d50b34eac9fa4 [ 614.387392] env[64020]: DEBUG nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 614.389226] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 5369a9f2159547c8b8ff414e25ee812e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 614.419912] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "d4d0d50d-8479-47f7-ba29-114ef10764cc" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 614.420172] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "d4d0d50d-8479-47f7-ba29-114ef10764cc" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 614.436845] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5369a9f2159547c8b8ff414e25ee812e [ 614.476859] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407823, 'name': ReconfigVM_Task, 'duration_secs': 0.30581} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 614.477456] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Reconfigured VM instance instance-00000009 to attach disk [datastore1] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 614.478255] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-0fc62f1b-76d3-4507-9c5a-b26e544a0f6a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 614.485713] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 614.485713] env[64020]: value = "task-407824" [ 614.485713] env[64020]: _type = "Task" [ 614.485713] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 614.497280] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407824, 'name': Rename_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 614.612990] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 614.613653] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 32d53d2a15cd44f48031bb6e047a0837 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 614.625969] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 32d53d2a15cd44f48031bb6e047a0837 [ 614.672096] env[64020]: INFO nova.scheduler.client.report [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Deleted allocations for instance 4d9f1765-f282-4a36-9020-7d0c1fbcc5f2 [ 614.680033] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg c7bb79d559344a06bd69623a37c66987 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 614.693095] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c7bb79d559344a06bd69623a37c66987 [ 614.781615] env[64020]: DEBUG nova.network.neutron [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 614.782296] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 1d09d0971c1241869ee9d8c5cd545d06 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 614.791804] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d09d0971c1241869ee9d8c5cd545d06 [ 614.880165] env[64020]: ERROR nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 57a9791c-3911-480c-bb77-c7950afb78c3, please check neutron logs for more information. [ 614.880165] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 614.880165] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 614.880165] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 614.880165] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 614.880165] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 614.880165] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 614.880165] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 614.880165] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 614.880165] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 614.880165] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 614.880165] env[64020]: ERROR nova.compute.manager raise self.value [ 614.880165] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 614.880165] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 614.880165] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 614.880165] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 614.880738] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 614.880738] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 614.880738] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 57a9791c-3911-480c-bb77-c7950afb78c3, please check neutron logs for more information. [ 614.880738] env[64020]: ERROR nova.compute.manager [ 614.880738] env[64020]: Traceback (most recent call last): [ 614.880738] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 614.880738] env[64020]: listener.cb(fileno) [ 614.880738] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 614.880738] env[64020]: result = function(*args, **kwargs) [ 614.880738] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 614.880738] env[64020]: return func(*args, **kwargs) [ 614.880738] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 614.880738] env[64020]: raise e [ 614.880738] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 614.880738] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 614.880738] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 614.880738] env[64020]: created_port_ids = self._update_ports_for_instance( [ 614.880738] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 614.880738] env[64020]: with excutils.save_and_reraise_exception(): [ 614.880738] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 614.880738] env[64020]: self.force_reraise() [ 614.880738] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 614.880738] env[64020]: raise self.value [ 614.880738] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 614.880738] env[64020]: updated_port = self._update_port( [ 614.880738] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 614.880738] env[64020]: _ensure_no_port_binding_failure(port) [ 614.880738] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 614.880738] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 614.881661] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 57a9791c-3911-480c-bb77-c7950afb78c3, please check neutron logs for more information. [ 614.881661] env[64020]: Removing descriptor: 17 [ 614.881661] env[64020]: ERROR nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 57a9791c-3911-480c-bb77-c7950afb78c3, please check neutron logs for more information. [ 614.881661] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Traceback (most recent call last): [ 614.881661] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 614.881661] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] yield resources [ 614.881661] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 614.881661] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] self.driver.spawn(context, instance, image_meta, [ 614.881661] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 614.881661] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 614.881661] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 614.881661] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] vm_ref = self.build_virtual_machine(instance, [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] vif_infos = vmwarevif.get_vif_info(self._session, [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] for vif in network_info: [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] return self._sync_wrapper(fn, *args, **kwargs) [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] self.wait() [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] self[:] = self._gt.wait() [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] return self._exit_event.wait() [ 614.882350] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] result = hub.switch() [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] return self.greenlet.switch() [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] result = function(*args, **kwargs) [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] return func(*args, **kwargs) [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] raise e [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] nwinfo = self.network_api.allocate_for_instance( [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 614.882990] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] created_port_ids = self._update_ports_for_instance( [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] with excutils.save_and_reraise_exception(): [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] self.force_reraise() [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] raise self.value [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] updated_port = self._update_port( [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] _ensure_no_port_binding_failure(port) [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 614.883627] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] raise exception.PortBindingFailed(port_id=port['id']) [ 614.884221] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] nova.exception.PortBindingFailed: Binding failed for port 57a9791c-3911-480c-bb77-c7950afb78c3, please check neutron logs for more information. [ 614.884221] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] [ 614.884221] env[64020]: INFO nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Terminating instance [ 614.887839] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Acquiring lock "refresh_cache-1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 614.887839] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Acquired lock "refresh_cache-1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 614.887839] env[64020]: DEBUG nova.network.neutron [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 614.887839] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 3bb8abfac7c04d428911c48cc6e12ddd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 614.897284] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3bb8abfac7c04d428911c48cc6e12ddd [ 614.912126] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 614.995511] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407824, 'name': Rename_Task, 'duration_secs': 0.129106} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 614.995802] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 614.996057] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-4299c5c4-35d1-48ff-84de-ac0af17aa38c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 615.001938] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 615.001938] env[64020]: value = "task-407825" [ 615.001938] env[64020]: _type = "Task" [ 615.001938] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 615.017006] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407825, 'name': PowerOnVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 615.118290] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 615.118417] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance b18fca61-2b17-4758-824d-e02d2cb342d8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 615.118480] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 7529a478-7efe-4dff-8429-810549f56bd7 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 615.118607] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 5290c09b-d77c-4d27-b367-9de52f54d16a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 615.118727] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance b3440b55-2469-46fa-ac2c-3e207bf530ec actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 615.118827] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 615.119442] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 108682cdeb4445b89b8ccb10914a9a0e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 615.142013] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 108682cdeb4445b89b8ccb10914a9a0e [ 615.182426] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b703d52-c5f5-42f5-97af-2381182044b4 tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "4d9f1765-f282-4a36-9020-7d0c1fbcc5f2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 32.630s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 615.183292] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg bfddccf39f06422f86f887356470dade in queue reply_57893177120949e6a93cb88e15cd42b4 [ 615.200768] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bfddccf39f06422f86f887356470dade [ 615.289842] env[64020]: INFO nova.compute.manager [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3] Took 1.05 seconds to deallocate network for instance. [ 615.291859] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 21ecc50759c84620bc67048009b58509 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 615.340440] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21ecc50759c84620bc67048009b58509 [ 615.411124] env[64020]: DEBUG nova.network.neutron [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 615.479027] env[64020]: DEBUG nova.network.neutron [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 615.479724] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg ccb6f30f67624b69971b8ce9d79c2c46 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 615.488388] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ccb6f30f67624b69971b8ce9d79c2c46 [ 615.516876] env[64020]: DEBUG oslo_vmware.api [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407825, 'name': PowerOnVM_Task, 'duration_secs': 0.434007} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 615.517223] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 615.517469] env[64020]: INFO nova.compute.manager [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Took 8.69 seconds to spawn the instance on the hypervisor. [ 615.517701] env[64020]: DEBUG nova.compute.manager [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 615.518971] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7662b00-abd9-4dad-9e9b-afb1d57a6fb2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 615.528546] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg bbd4754f03f54393b7b0a7add7e2e5fd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 615.547556] env[64020]: DEBUG nova.compute.manager [req-420667b4-9c7e-4811-8276-3642d9581a4c req-b04fc626-30ac-4c4d-8d4e-ce2ca57775c0 service nova] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Received event network-changed-57a9791c-3911-480c-bb77-c7950afb78c3 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 615.547741] env[64020]: DEBUG nova.compute.manager [req-420667b4-9c7e-4811-8276-3642d9581a4c req-b04fc626-30ac-4c4d-8d4e-ce2ca57775c0 service nova] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Refreshing instance network info cache due to event network-changed-57a9791c-3911-480c-bb77-c7950afb78c3. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 615.547923] env[64020]: DEBUG oslo_concurrency.lockutils [req-420667b4-9c7e-4811-8276-3642d9581a4c req-b04fc626-30ac-4c4d-8d4e-ce2ca57775c0 service nova] Acquiring lock "refresh_cache-1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 615.590767] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bbd4754f03f54393b7b0a7add7e2e5fd [ 615.623780] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 65be7d88-0c12-4ad3-adc7-d089a1b91c28 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 615.623780] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg ccd24da2bfc14d07a5376d711ee8970d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 615.639974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ccd24da2bfc14d07a5376d711ee8970d [ 615.686331] env[64020]: DEBUG nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 615.688285] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 8d6f3f2c0cae423cb82b09559747f9bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 615.725702] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d6f3f2c0cae423cb82b09559747f9bc [ 615.796864] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg b730b4cfef9e4eb797efa8e928a79f69 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 615.835700] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b730b4cfef9e4eb797efa8e928a79f69 [ 615.982379] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Releasing lock "refresh_cache-1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 615.982860] env[64020]: DEBUG nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 615.983093] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 615.983466] env[64020]: DEBUG oslo_concurrency.lockutils [req-420667b4-9c7e-4811-8276-3642d9581a4c req-b04fc626-30ac-4c4d-8d4e-ce2ca57775c0 service nova] Acquired lock "refresh_cache-1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 615.983653] env[64020]: DEBUG nova.network.neutron [req-420667b4-9c7e-4811-8276-3642d9581a4c req-b04fc626-30ac-4c4d-8d4e-ce2ca57775c0 service nova] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Refreshing network info cache for port 57a9791c-3911-480c-bb77-c7950afb78c3 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 615.984122] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-420667b4-9c7e-4811-8276-3642d9581a4c req-b04fc626-30ac-4c4d-8d4e-ce2ca57775c0 service nova] Expecting reply to msg 39375993d06040b39f7d2421d37acd56 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 615.984938] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2e7ac528-cf1d-42a2-802d-dec466c034ea {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 615.994253] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d8f7332-b71c-41e0-b866-4fa179b44a18 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 616.006179] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 39375993d06040b39f7d2421d37acd56 [ 616.020935] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c could not be found. [ 616.020935] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 616.020935] env[64020]: INFO nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Took 0.04 seconds to destroy the instance on the hypervisor. [ 616.020935] env[64020]: DEBUG oslo.service.loopingcall [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 616.021178] env[64020]: DEBUG nova.compute.manager [-] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 616.021178] env[64020]: DEBUG nova.network.neutron [-] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 616.038559] env[64020]: INFO nova.compute.manager [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Took 22.47 seconds to build instance. [ 616.039169] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg ccfdd5b91bfe4f1b8fbe3533c0c34378 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 616.055637] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ccfdd5b91bfe4f1b8fbe3533c0c34378 [ 616.126047] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance e5b12149-0e4c-41b2-b726-f8a55452669a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 616.126643] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 714580b9a8034b1db3c1fe1ae3cff0a4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 616.136550] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 714580b9a8034b1db3c1fe1ae3cff0a4 [ 616.142270] env[64020]: DEBUG nova.network.neutron [-] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 616.142766] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg fa9ea419d51d470e93ce5c52369cec59 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 616.150103] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fa9ea419d51d470e93ce5c52369cec59 [ 616.214926] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 616.322358] env[64020]: INFO nova.scheduler.client.report [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Deleted allocations for instance 60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3 [ 616.329707] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 4aa1435d6bc84ed280bb65b287a795c3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 616.340342] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4aa1435d6bc84ed280bb65b287a795c3 [ 616.541152] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b34f6e8b-7e92-42d3-acc3-4bbb5503ff80 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "b3440b55-2469-46fa-ac2c-3e207bf530ec" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 23.981s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 616.541769] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 32c4bfbee73e4f4f9cbe8fbf5d1466bb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 616.559101] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 32c4bfbee73e4f4f9cbe8fbf5d1466bb [ 616.615334] env[64020]: DEBUG nova.network.neutron [req-420667b4-9c7e-4811-8276-3642d9581a4c req-b04fc626-30ac-4c4d-8d4e-ce2ca57775c0 service nova] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 616.629562] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance a98ba289-b065-4cef-a7bd-7b7cd07b141d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 616.630152] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 5edf2caa98ab4df987db5a27157d7eba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 616.642496] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5edf2caa98ab4df987db5a27157d7eba [ 616.644241] env[64020]: DEBUG nova.network.neutron [-] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 616.644649] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a90ea6a2266b471eb65521a814451c01 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 616.653944] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a90ea6a2266b471eb65521a814451c01 [ 616.832383] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2e7d6fd6-670b-4569-9951-bbd48d612408 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "60ebafe1-2e3e-4c6c-ba4d-1e9c9c9454b3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 32.077s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 616.833110] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg 50b95c6cdc034469a033351cde167b53 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 616.845908] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50b95c6cdc034469a033351cde167b53 [ 617.003734] env[64020]: DEBUG nova.network.neutron [req-420667b4-9c7e-4811-8276-3642d9581a4c req-b04fc626-30ac-4c4d-8d4e-ce2ca57775c0 service nova] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 617.004444] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-420667b4-9c7e-4811-8276-3642d9581a4c req-b04fc626-30ac-4c4d-8d4e-ce2ca57775c0 service nova] Expecting reply to msg 713b7ab0c2ac428dada4ba42b8c71477 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 617.015042] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 713b7ab0c2ac428dada4ba42b8c71477 [ 617.044472] env[64020]: DEBUG nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 617.046204] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 353913765e764a30b5d82bf1eed41ad8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 617.097913] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 353913765e764a30b5d82bf1eed41ad8 [ 617.132768] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance ae57ce0e-46b5-4490-af3f-4b2537bf316a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 617.133315] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 15f67f6f3b754d3faf6d6e8526a7aad8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 617.145767] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 15f67f6f3b754d3faf6d6e8526a7aad8 [ 617.152061] env[64020]: INFO nova.compute.manager [-] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Took 1.13 seconds to deallocate network for instance. [ 617.152785] env[64020]: DEBUG nova.compute.claims [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 617.152955] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.338633] env[64020]: DEBUG nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 617.338633] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg 9e272c73a7c6411581d25194315a56e3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 617.370591] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e272c73a7c6411581d25194315a56e3 [ 617.507137] env[64020]: DEBUG oslo_concurrency.lockutils [req-420667b4-9c7e-4811-8276-3642d9581a4c req-b04fc626-30ac-4c4d-8d4e-ce2ca57775c0 service nova] Releasing lock "refresh_cache-1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 617.567203] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.640159] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 76de4647-9281-4c49-b4f1-16f0ea2ff313 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 617.640159] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 6cec22bf5ecd4160a8d23be40c4a2439 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 617.648665] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6cec22bf5ecd4160a8d23be40c4a2439 [ 617.866645] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.143964] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance bfb84a86-8de0-4b2d-9355-85e0d8b5eba2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 618.143964] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 20c903f0a8ee408c84eeb091178a9c06 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 618.157850] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20c903f0a8ee408c84eeb091178a9c06 [ 618.204488] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquiring lock "7437f3e0-0112-4640-9076-5f5f864ec230" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.204709] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "7437f3e0-0112-4640-9076-5f5f864ec230" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 618.451023] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "af8b2b66-c2d6-4bb6-a266-52cb82bac3be" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.451303] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "af8b2b66-c2d6-4bb6-a266-52cb82bac3be" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 618.645470] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 618c3b06-1fd8-45d3-9c59-61c0e202a299 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 618.646123] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 765e4ffa47aa4e458d7c515f9aceb512 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 618.656411] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 765e4ffa47aa4e458d7c515f9aceb512 [ 619.148855] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 4478746a-11a6-480d-84c1-c6f04ab33505 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 619.149683] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 82d449b40d814925b2356471d6cfe4d9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 619.161162] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82d449b40d814925b2356471d6cfe4d9 [ 619.258643] env[64020]: DEBUG nova.compute.manager [req-69e51bb0-d9c3-4a71-9f3a-7ebe981acde3 req-c366e9f7-7088-4f76-9aa7-e968ea00e0fe service nova] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Received event network-vif-deleted-57a9791c-3911-480c-bb77-c7950afb78c3 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 619.496263] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Acquiring lock "583b323c-a4e1-4431-9bc0-d9f4b7354cb6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 619.496512] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Lock "583b323c-a4e1-4431-9bc0-d9f4b7354cb6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 619.537206] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 72f65309f55b4a699c5b2715904ced72 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 619.547627] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72f65309f55b4a699c5b2715904ced72 [ 619.655235] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 619.655235] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg c82361d001c34bb698e953bad0135f34 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 619.669750] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c82361d001c34bb698e953bad0135f34 [ 620.040329] env[64020]: INFO nova.compute.manager [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Rebuilding instance [ 620.104131] env[64020]: DEBUG nova.compute.manager [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 620.105858] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eee546a2-e8fe-4c6e-a52d-32a1138e8d9b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.117718] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg b1260b6b9ce243eb9f50205a95af82fc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 620.158366] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance cea1a347-7f1a-4090-b3ad-54ccfbdfea2a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 620.160486] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 10d5ede7cd5c4667928059d4c41a6337 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 620.176498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10d5ede7cd5c4667928059d4c41a6337 [ 620.209392] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1260b6b9ce243eb9f50205a95af82fc [ 620.621170] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 620.625077] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-8de1347a-6f3e-4765-a67c-9a5465918170 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.628549] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 620.628549] env[64020]: value = "task-407826" [ 620.628549] env[64020]: _type = "Task" [ 620.628549] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 620.637058] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407826, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 620.671304] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 5ac25e1d-007a-4eaa-811c-655b54efd7ec has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 620.672084] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 86cc6cb259ee4f2cbaa7570bd61a82d1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 620.683628] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86cc6cb259ee4f2cbaa7570bd61a82d1 [ 620.916097] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "8368f535-f3d4-4b03-98ef-20b0bf91d25c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 620.916359] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "8368f535-f3d4-4b03-98ef-20b0bf91d25c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 621.142223] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407826, 'name': PowerOffVM_Task, 'duration_secs': 0.117977} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 621.142520] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 621.142743] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 621.143580] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c59d151b-2e23-424e-9598-16db906a37ad {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.150466] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 621.150708] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-dd588fe0-5aed-46f9-b1b3-f74fdb3f9dae {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.175901] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance d4d0d50d-8479-47f7-ba29-114ef10764cc has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 621.176596] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Total usable vcpus: 48, total allocated vcpus: 5 {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 621.176596] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1472MB phys_disk=200GB used_disk=5GB total_vcpus=48 used_vcpus=5 pci_stats=[] {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 621.180179] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 621.180383] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Deleting contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 621.180556] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleting the datastore file [datastore1] b3440b55-2469-46fa-ac2c-3e207bf530ec {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 621.180793] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-c5f6ebe1-e701-47f0-bba0-fea8ffdfc6d7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.187152] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 621.187152] env[64020]: value = "task-407828" [ 621.187152] env[64020]: _type = "Task" [ 621.187152] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 621.198039] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407828, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 621.506276] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5c31304b-fb2e-483d-b857-ca71e023d753 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.514314] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b837268-744a-4f6e-8e53-6b7254f41c42 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.546769] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ea68bac-05f2-4a22-bdc7-eb493cd6d778 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.554690] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e8af6ae-6db2-4417-8a7c-8ed24533468c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.568694] env[64020]: DEBUG nova.compute.provider_tree [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 621.569171] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 4e9e6d5138274c1a8fe4ccd7d50a3e08 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 621.576205] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e9e6d5138274c1a8fe4ccd7d50a3e08 [ 621.703590] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407828, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.107} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 621.703890] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 621.704200] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Deleted contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 621.704403] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 621.706036] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 0ec00422c98e472fa2077545e7a3966a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 621.772806] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0ec00422c98e472fa2077545e7a3966a [ 622.072323] env[64020]: DEBUG nova.scheduler.client.report [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 622.074747] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 5a020d50c2cf459797bf4a99a31fc0de in queue reply_57893177120949e6a93cb88e15cd42b4 [ 622.093465] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a020d50c2cf459797bf4a99a31fc0de [ 622.211793] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg ddd3bd932e574e9cb51c83b784cb9cc5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 622.248872] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ddd3bd932e574e9cb51c83b784cb9cc5 [ 622.576809] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=64020) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 622.577148] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 9.507s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.577318] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.152s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.579141] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 730428f575e1446ab7d734671c6f072f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 622.647674] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 730428f575e1446ab7d734671c6f072f [ 622.710455] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "979aa8d0-1e9b-4bbb-b507-31f72542d8d0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.710736] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "979aa8d0-1e9b-4bbb-b507-31f72542d8d0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.749011] env[64020]: DEBUG nova.virt.hardware [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 622.749268] env[64020]: DEBUG nova.virt.hardware [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 622.749417] env[64020]: DEBUG nova.virt.hardware [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 622.749587] env[64020]: DEBUG nova.virt.hardware [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 622.749723] env[64020]: DEBUG nova.virt.hardware [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 622.749934] env[64020]: DEBUG nova.virt.hardware [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 622.750129] env[64020]: DEBUG nova.virt.hardware [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 622.750252] env[64020]: DEBUG nova.virt.hardware [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 622.750367] env[64020]: DEBUG nova.virt.hardware [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 622.750481] env[64020]: DEBUG nova.virt.hardware [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 622.750639] env[64020]: DEBUG nova.virt.hardware [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 622.751551] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5db05f26-9149-4899-b5b6-cb597d2f9423 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.765221] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e457a27-eded-4790-9a0a-a6cdc656e8e5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.781898] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 622.787426] env[64020]: DEBUG oslo.service.loopingcall [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 622.787942] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 622.788171] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-462a5f13-b2a4-4599-be09-5479f68a57b0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.809950] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 622.809950] env[64020]: value = "task-407829" [ 622.809950] env[64020]: _type = "Task" [ 622.809950] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 622.825190] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407829, 'name': CreateVM_Task} progress is 6%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 623.034627] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "0d97356d-3abf-4207-842d-6e2d468ef97c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 623.034857] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "0d97356d-3abf-4207-842d-6e2d468ef97c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 623.321625] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407829, 'name': CreateVM_Task} progress is 99%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 623.516289] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2bdcb687-db27-4af0-8129-5c2034947333 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 623.523742] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ceb91e6-a571-4519-bc78-7d9ce2fbb7f5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 623.559204] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-201f39bf-c24a-4a94-85ba-57b768d4026e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 623.570769] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79ded3f7-0306-45c8-9d14-d748e52bf9f3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 623.586684] env[64020]: DEBUG nova.compute.provider_tree [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 623.587245] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg ba7be874e6954e13bae1b53cf1ef9766 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 623.596093] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba7be874e6954e13bae1b53cf1ef9766 [ 623.830295] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407829, 'name': CreateVM_Task, 'duration_secs': 0.567881} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 623.830295] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 623.830295] env[64020]: DEBUG oslo_vmware.service [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c17ec65-6d7b-43c1-8240-98422f80a294 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 623.841695] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 623.841695] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquired lock "[datastore2] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 623.841695] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 623.841695] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-987e0071-aee8-4f1a-92f2-0f41d838b401 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 623.844609] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 623.844609] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]521a7b29-11c7-bdcb-8fab-7a167fdf7f9c" [ 623.844609] env[64020]: _type = "Task" [ 623.844609] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 623.858462] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]521a7b29-11c7-bdcb-8fab-7a167fdf7f9c, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 624.093581] env[64020]: DEBUG nova.scheduler.client.report [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 624.096019] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg b1aa51a2fc7b40b99fbc3ae86aa4833b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 624.112399] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1aa51a2fc7b40b99fbc3ae86aa4833b [ 624.358370] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Releasing lock "[datastore2] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 624.358370] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 624.358370] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 624.358370] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquired lock "[datastore2] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 624.358675] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 624.358759] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0d772be2-fbf3-4a18-9f9c-9d67bd932d9c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 624.368685] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 624.368685] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 624.368685] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c842c612-1835-4ea1-b89a-c7d67660b7be {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 624.374275] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f6d5a854-05bc-4576-8f93-1f6ac143d4fd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 624.379242] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 624.379242] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]521fa75b-9951-34e3-1c58-301da537aa52" [ 624.379242] env[64020]: _type = "Task" [ 624.379242] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 624.386392] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]521fa75b-9951-34e3-1c58-301da537aa52, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 624.604629] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.027s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 624.605272] env[64020]: ERROR nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3d243fb0-6d5c-4646-b594-0f1677cf790f, please check neutron logs for more information. [ 624.605272] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Traceback (most recent call last): [ 624.605272] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 624.605272] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] self.driver.spawn(context, instance, image_meta, [ 624.605272] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 624.605272] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 624.605272] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 624.605272] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] vm_ref = self.build_virtual_machine(instance, [ 624.605272] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 624.605272] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] vif_infos = vmwarevif.get_vif_info(self._session, [ 624.605272] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] for vif in network_info: [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] return self._sync_wrapper(fn, *args, **kwargs) [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] self.wait() [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] self[:] = self._gt.wait() [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] return self._exit_event.wait() [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] result = hub.switch() [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 624.605698] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] return self.greenlet.switch() [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] result = function(*args, **kwargs) [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] return func(*args, **kwargs) [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] raise e [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] nwinfo = self.network_api.allocate_for_instance( [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] created_port_ids = self._update_ports_for_instance( [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] with excutils.save_and_reraise_exception(): [ 624.606114] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 624.606530] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] self.force_reraise() [ 624.606530] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 624.606530] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] raise self.value [ 624.606530] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 624.606530] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] updated_port = self._update_port( [ 624.606530] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 624.606530] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] _ensure_no_port_binding_failure(port) [ 624.606530] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 624.606530] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] raise exception.PortBindingFailed(port_id=port['id']) [ 624.606530] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] nova.exception.PortBindingFailed: Binding failed for port 3d243fb0-6d5c-4646-b594-0f1677cf790f, please check neutron logs for more information. [ 624.606530] env[64020]: ERROR nova.compute.manager [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] [ 624.606884] env[64020]: DEBUG nova.compute.utils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Binding failed for port 3d243fb0-6d5c-4646-b594-0f1677cf790f, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 624.607244] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 22.199s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 624.609013] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg a58d7da2dbb341be8879a2d1eb1a43dd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 624.610223] env[64020]: DEBUG nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Build of instance b18fca61-2b17-4758-824d-e02d2cb342d8 was re-scheduled: Binding failed for port 3d243fb0-6d5c-4646-b594-0f1677cf790f, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 624.610674] env[64020]: DEBUG nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 624.610864] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Acquiring lock "refresh_cache-b18fca61-2b17-4758-824d-e02d2cb342d8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 624.611003] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Acquired lock "refresh_cache-b18fca61-2b17-4758-824d-e02d2cb342d8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 624.611208] env[64020]: DEBUG nova.network.neutron [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 624.611578] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 0bc4559a3f3e41e3aa63f93f96f47402 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 624.619318] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0bc4559a3f3e41e3aa63f93f96f47402 [ 624.662389] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Acquiring lock "727e7787-68b7-441c-a31d-6f0796c33be0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 624.662633] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Lock "727e7787-68b7-441c-a31d-6f0796c33be0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 624.673826] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a58d7da2dbb341be8879a2d1eb1a43dd [ 624.889616] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Preparing fetch location {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 624.889897] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Creating directory with path [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 624.890115] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-6cc61cb1-eefa-4e5d-8cf5-1601432d0a19 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 624.910187] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Created directory with path [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 624.910396] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Fetch image to [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 624.910563] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Downloading image file data 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 to [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk on the data store datastore2 {{(pid=64020) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 624.911433] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-731736cd-3b64-44c4-8b05-0bcf45b555aa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 624.919815] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7266e7b-0a55-4b78-8a0e-0892f1b76ef2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 624.930232] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42a6a705-c260-48ba-a1be-cdb8c0774915 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 624.965133] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7faf3e7e-9a04-4214-a24a-3d5420004be3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 624.971448] env[64020]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-7f16a4c9-a63f-4780-8fc9-06fb89e5fff1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.055519] env[64020]: DEBUG nova.virt.vmwareapi.images [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Downloading image file data 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 to the data store datastore2 {{(pid=64020) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 625.126541] env[64020]: DEBUG oslo_vmware.rw_handles [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=64020) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 625.196082] env[64020]: DEBUG nova.network.neutron [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 625.427513] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Acquiring lock "5e43d3ea-740c-4569-be82-3bb7082fa8ea" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 625.427748] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Lock "5e43d3ea-740c-4569-be82-3bb7082fa8ea" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 625.432488] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquiring lock "866f8ba4-faad-4883-80ca-4c4dedb2b50c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 625.432700] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "866f8ba4-faad-4883-80ca-4c4dedb2b50c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 625.563292] env[64020]: DEBUG nova.network.neutron [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 625.563860] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 177e4bfb7339491eb02893b9e88e2f53 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 625.578214] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 177e4bfb7339491eb02893b9e88e2f53 [ 625.664094] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae983592-c6ef-4c83-a8b7-6dce0c3a2714 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.671844] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5274440-0218-46d3-a898-8996083a97a0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.705875] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f0a2b45-b3e0-4d6a-8793-60a72e609f29 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.714360] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf9f9aa2-40f7-4099-9471-19bd8aa3812e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.728847] env[64020]: DEBUG nova.compute.provider_tree [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 625.729202] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 74474e301bde45cf919d4acd1df8bdc0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 625.737223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74474e301bde45cf919d4acd1df8bdc0 [ 625.860100] env[64020]: DEBUG oslo_vmware.rw_handles [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Completed reading data from the image iterator. {{(pid=64020) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 625.860100] env[64020]: DEBUG oslo_vmware.rw_handles [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=64020) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 625.931375] env[64020]: DEBUG nova.virt.vmwareapi.images [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Downloaded image file data 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 to vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk on the data store datastore2 {{(pid=64020) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 625.931375] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Caching image {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 625.931375] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Copying Virtual Disk [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk to [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 625.931375] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-186b1319-e182-498c-a36d-5ed86a106d47 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.936922] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 625.936922] env[64020]: value = "task-407834" [ 625.936922] env[64020]: _type = "Task" [ 625.936922] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 625.945991] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407834, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 626.078522] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Releasing lock "refresh_cache-b18fca61-2b17-4758-824d-e02d2cb342d8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 626.078786] env[64020]: DEBUG nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 626.078969] env[64020]: DEBUG nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 626.079150] env[64020]: DEBUG nova.network.neutron [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 626.241950] env[64020]: DEBUG nova.scheduler.client.report [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 626.241950] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 3d2a923d23754fc69e81c3a7d52f8605 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 626.252152] env[64020]: DEBUG nova.network.neutron [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 626.252836] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg b775d761c8e84dc7b2061d7f1e2f9912 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 626.262211] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b775d761c8e84dc7b2061d7f1e2f9912 [ 626.265657] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d2a923d23754fc69e81c3a7d52f8605 [ 626.447843] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407834, 'name': CopyVirtualDisk_Task} progress is 100%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 626.748440] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.140s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 626.748440] env[64020]: ERROR nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 5820ea5a-bebc-46c4-a6a9-3551052bf734, please check neutron logs for more information. [ 626.748440] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Traceback (most recent call last): [ 626.748440] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 626.748440] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] self.driver.spawn(context, instance, image_meta, [ 626.748440] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 626.748440] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 626.748440] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 626.748440] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] vm_ref = self.build_virtual_machine(instance, [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] vif_infos = vmwarevif.get_vif_info(self._session, [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] for vif in network_info: [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] return self._sync_wrapper(fn, *args, **kwargs) [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] self.wait() [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] self[:] = self._gt.wait() [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] return self._exit_event.wait() [ 626.748835] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] result = hub.switch() [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] return self.greenlet.switch() [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] result = function(*args, **kwargs) [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] return func(*args, **kwargs) [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] raise e [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] nwinfo = self.network_api.allocate_for_instance( [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 626.749208] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] created_port_ids = self._update_ports_for_instance( [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] with excutils.save_and_reraise_exception(): [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] self.force_reraise() [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] raise self.value [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] updated_port = self._update_port( [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] _ensure_no_port_binding_failure(port) [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 626.749555] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] raise exception.PortBindingFailed(port_id=port['id']) [ 626.749879] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] nova.exception.PortBindingFailed: Binding failed for port 5820ea5a-bebc-46c4-a6a9-3551052bf734, please check neutron logs for more information. [ 626.749879] env[64020]: ERROR nova.compute.manager [instance: 7529a478-7efe-4dff-8429-810549f56bd7] [ 626.749879] env[64020]: DEBUG nova.compute.utils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Binding failed for port 5820ea5a-bebc-46c4-a6a9-3551052bf734, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 626.750364] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 21.224s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 626.754456] env[64020]: INFO nova.compute.claims [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 626.756973] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 44742406779e48b79df84d82fdad00e9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 626.756973] env[64020]: DEBUG nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Build of instance 7529a478-7efe-4dff-8429-810549f56bd7 was re-scheduled: Binding failed for port 5820ea5a-bebc-46c4-a6a9-3551052bf734, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 626.757373] env[64020]: DEBUG nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 626.757373] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Acquiring lock "refresh_cache-7529a478-7efe-4dff-8429-810549f56bd7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 626.757483] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Acquired lock "refresh_cache-7529a478-7efe-4dff-8429-810549f56bd7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 626.757775] env[64020]: DEBUG nova.network.neutron [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 626.757909] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 028bd03f8c2c4c8a890f6205a18f5c90 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 626.758998] env[64020]: DEBUG nova.network.neutron [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 626.759405] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg cc98006f4d284a419195d143beac525a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 626.778116] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 028bd03f8c2c4c8a890f6205a18f5c90 [ 626.818521] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cc98006f4d284a419195d143beac525a [ 626.837040] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44742406779e48b79df84d82fdad00e9 [ 626.948082] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407834, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.696482} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 626.948363] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Copied Virtual Disk [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk to [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 626.948519] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleting the datastore file [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/tmp-sparse.vmdk {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 626.948768] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-b0534619-3bf5-4014-a0c8-9956a6b5025b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 626.954747] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 626.954747] env[64020]: value = "task-407835" [ 626.954747] env[64020]: _type = "Task" [ 626.954747] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 626.962806] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407835, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 627.265183] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg d29ec04c0c2d476097560a5815793b6f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 627.276384] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d29ec04c0c2d476097560a5815793b6f [ 627.289580] env[64020]: INFO nova.compute.manager [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] [instance: b18fca61-2b17-4758-824d-e02d2cb342d8] Took 1.21 seconds to deallocate network for instance. [ 627.291849] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 48cc4bcf8da54ea5a20004690ecf1d98 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 627.300627] env[64020]: DEBUG nova.network.neutron [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 627.332509] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 48cc4bcf8da54ea5a20004690ecf1d98 [ 627.466683] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407835, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.043822} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 627.470056] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 627.470417] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Moving file from [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 to [datastore2] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234. {{(pid=64020) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:334}} [ 627.473058] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MoveDatastoreFile_Task with opID=oslo.vmware-f41dc1b2-f076-4f27-93b7-17d79708c9bf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 627.484305] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 627.484305] env[64020]: value = "task-407837" [ 627.484305] env[64020]: _type = "Task" [ 627.484305] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 627.494078] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407837, 'name': MoveDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 627.596147] env[64020]: DEBUG nova.network.neutron [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 627.596147] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 4d12db9647724a829128be727a00351d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 627.606256] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d12db9647724a829128be727a00351d [ 627.780210] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Acquiring lock "371d5fb1-f1b6-41de-a4e6-88e8137f4be9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.780995] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Lock "371d5fb1-f1b6-41de-a4e6-88e8137f4be9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.798919] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg d811eb5d22554c2aa51eae4c61c2a728 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 627.852200] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d811eb5d22554c2aa51eae4c61c2a728 [ 627.995334] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407837, 'name': MoveDatastoreFile_Task, 'duration_secs': 0.068016} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 627.995586] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] File moved {{(pid=64020) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:346}} [ 627.995718] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Cleaning up location [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:640}} [ 627.996098] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleting the datastore file [datastore2] vmware_temp/772adf44-5fa7-4322-8a2e-aeb61679c33f {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 627.996475] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-83c403b9-1920-46c4-bffe-3915260c8e7c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.010134] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 628.010134] env[64020]: value = "task-407838" [ 628.010134] env[64020]: _type = "Task" [ 628.010134] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 628.019823] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407838, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 628.097896] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Releasing lock "refresh_cache-7529a478-7efe-4dff-8429-810549f56bd7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 628.098123] env[64020]: DEBUG nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 628.098298] env[64020]: DEBUG nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 628.098490] env[64020]: DEBUG nova.network.neutron [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 628.131956] env[64020]: DEBUG nova.network.neutron [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 628.132558] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg c99d7563c1164bd4acf5ad139998a4fa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 628.141985] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c99d7563c1164bd4acf5ad139998a4fa [ 628.218513] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d00b4d57-9698-4985-a290-6faab88941a4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.224106] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-270bc657-9590-4957-a466-44fbd5afb2be {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.262089] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-913a4d3b-d70e-4d83-b3cf-493b2f953509 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.273222] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc74643c-34be-4818-9831-771cafd7ffde {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.290902] env[64020]: DEBUG nova.compute.provider_tree [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 628.291779] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 88c783881bd5414bb56f482cad54c520 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 628.299229] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88c783881bd5414bb56f482cad54c520 [ 628.321318] env[64020]: INFO nova.scheduler.client.report [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Deleted allocations for instance b18fca61-2b17-4758-824d-e02d2cb342d8 [ 628.330098] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Expecting reply to msg 1209447eb7f24f00aa4226556bf69516 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 628.349483] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1209447eb7f24f00aa4226556bf69516 [ 628.521135] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407838, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.028303} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 628.521451] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 628.522361] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-bebce48a-cdef-45d7-a6b3-1912f8702f60 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.529023] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 628.529023] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5242548e-aae4-b088-a952-77781b3a51dc" [ 628.529023] env[64020]: _type = "Task" [ 628.529023] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 628.539536] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5242548e-aae4-b088-a952-77781b3a51dc, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 628.634970] env[64020]: DEBUG nova.network.neutron [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 628.635498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 5ee7d871357640c488602d5a8a3efbb2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 628.644668] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ee7d871357640c488602d5a8a3efbb2 [ 628.795260] env[64020]: DEBUG nova.scheduler.client.report [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 628.797606] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 66320beeb86c47ea95def43596834cf7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 628.810792] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66320beeb86c47ea95def43596834cf7 [ 628.832277] env[64020]: DEBUG oslo_concurrency.lockutils [None req-68ee0c23-c046-41c5-b23a-0a3da83ac269 tempest-ImagesOneServerNegativeTestJSON-1757012504 tempest-ImagesOneServerNegativeTestJSON-1757012504-project-member] Lock "b18fca61-2b17-4758-824d-e02d2cb342d8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 43.088s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 628.832881] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 9f8fd9ddcae14f88bdf0ed900219d8a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 628.849593] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f8fd9ddcae14f88bdf0ed900219d8a2 [ 629.041031] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5242548e-aae4-b088-a952-77781b3a51dc, 'name': SearchDatastore_Task, 'duration_secs': 0.009172} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 629.041031] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Releasing lock "[datastore2] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 629.041031] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore2] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 629.041031] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-2525e9b5-d4f0-44a6-9a42-7f9d557a8082 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 629.051003] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 629.051003] env[64020]: value = "task-407839" [ 629.051003] env[64020]: _type = "Task" [ 629.051003] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 629.059830] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407839, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 629.138521] env[64020]: INFO nova.compute.manager [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] [instance: 7529a478-7efe-4dff-8429-810549f56bd7] Took 1.04 seconds to deallocate network for instance. [ 629.140977] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 28e2540f7fe1488e89d05199e7c67708 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 629.210640] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28e2540f7fe1488e89d05199e7c67708 [ 629.300547] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.550s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 629.301057] env[64020]: DEBUG nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 629.302781] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg fff8b8899f784103bf8399cb511e86c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 629.307194] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 23.151s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 629.311448] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 332e64f044714821b9ab4c727231a77d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 629.317169] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "a26baa75-d36d-4c05-bde8-d39fe814191d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 629.317409] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "a26baa75-d36d-4c05-bde8-d39fe814191d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 629.334479] env[64020]: DEBUG nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 629.336635] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg e862011588924e28a629c6095cc0bbf8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 629.365180] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fff8b8899f784103bf8399cb511e86c1 [ 629.366049] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 332e64f044714821b9ab4c727231a77d [ 629.386000] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e862011588924e28a629c6095cc0bbf8 [ 629.562627] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407839, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.461581} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 629.563205] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore2] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 629.563624] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 629.563929] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-6911a54d-ccd0-4467-a323-a51c7e53c6b5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 629.575041] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 629.575041] env[64020]: value = "task-407841" [ 629.575041] env[64020]: _type = "Task" [ 629.575041] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 629.591167] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407841, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 629.648504] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 0db62d42672f472aaffdcdc915b75d5a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 629.687274] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0db62d42672f472aaffdcdc915b75d5a [ 629.808728] env[64020]: DEBUG nova.compute.utils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 629.809414] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 47da4282b2314ae4bfed7672289c562b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 629.810345] env[64020]: DEBUG nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 629.810517] env[64020]: DEBUG nova.network.neutron [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 629.820441] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47da4282b2314ae4bfed7672289c562b [ 629.861724] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 629.939816] env[64020]: DEBUG nova.policy [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e53a328bbb6f41e88a2dafd9247170de', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7c874d59a6eb4aa2a9e2c9c575cc24aa', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 630.043058] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "e01583c9-744c-47ea-b7cf-a46b046fc854" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 630.043058] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "e01583c9-744c-47ea-b7cf-a46b046fc854" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 630.090495] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "786a572c-adaa-4656-8b42-c510940dac59" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 630.090495] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "786a572c-adaa-4656-8b42-c510940dac59" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 630.090919] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407841, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.180218} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 630.092051] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 630.093566] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2df0a1f7-6d34-4d9a-a791-75e51d5a57d1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.116765] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Reconfiguring VM instance instance-00000009 to attach disk [datastore2] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 630.121550] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-bfa36164-67d8-4f14-81a8-54b59a21f1a2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.140304] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "8e8e1b50-fcae-41c6-8f5a-241692140cda" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 630.140656] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "8e8e1b50-fcae-41c6-8f5a-241692140cda" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 630.147946] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 630.147946] env[64020]: value = "task-407842" [ 630.147946] env[64020]: _type = "Task" [ 630.147946] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 630.168271] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407842, 'name': ReconfigVM_Task} progress is 10%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 630.182190] env[64020]: INFO nova.scheduler.client.report [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Deleted allocations for instance 7529a478-7efe-4dff-8429-810549f56bd7 [ 630.191687] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Expecting reply to msg 1f7bfe551a044b14b548f2d5c961e4e9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 630.213696] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f7bfe551a044b14b548f2d5c961e4e9 [ 630.313548] env[64020]: DEBUG nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 630.315564] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg dc88461137ce4e2eabf58e3370ca4ba5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 630.363411] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc88461137ce4e2eabf58e3370ca4ba5 [ 630.421766] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb5827a4-aa36-4b9d-af55-87f9182c609d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.430784] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0bbb83c-be74-41ee-b374-53d57cd0781c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.475680] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6fb367de-3168-4642-ad1e-06d8b186a4c9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.484667] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26f9ea90-959f-4d78-a963-4de7d27faa0f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.499611] env[64020]: DEBUG nova.compute.provider_tree [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 630.500157] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 32738d28151f4b6d9d0d93d07551c4b5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 630.508363] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 32738d28151f4b6d9d0d93d07551c4b5 [ 630.663768] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407842, 'name': ReconfigVM_Task, 'duration_secs': 0.295587} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 630.665738] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Reconfigured VM instance instance-00000009 to attach disk [datastore2] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 630.666461] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-a16a46dc-3db3-4797-a8fe-0f53402ddd88 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.678306] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 630.678306] env[64020]: value = "task-407843" [ 630.678306] env[64020]: _type = "Task" [ 630.678306] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 630.687455] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407843, 'name': Rename_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 630.691355] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1f09f1b6-4042-4d0e-9591-f9e2c6899697 tempest-FloatingIPsAssociationNegativeTestJSON-304767461 tempest-FloatingIPsAssociationNegativeTestJSON-304767461-project-member] Lock "7529a478-7efe-4dff-8429-810549f56bd7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 43.738s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 630.691831] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg ad8d44147d064376a9b3b0c1bc0ee972 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 630.703893] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad8d44147d064376a9b3b0c1bc0ee972 [ 630.821866] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 076a3ecff3814991aaef62439d41e4c6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 630.858634] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 076a3ecff3814991aaef62439d41e4c6 [ 631.003665] env[64020]: DEBUG nova.scheduler.client.report [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 631.006194] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg d70211a40c8d465f81c5a14163e4ffbf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 631.024046] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d70211a40c8d465f81c5a14163e4ffbf [ 631.055736] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "d808804b-a2d0-4225-beb5-7d91e0cff3ac" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 631.056045] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "d808804b-a2d0-4225-beb5-7d91e0cff3ac" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 631.087580] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 631.087792] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 631.181544] env[64020]: DEBUG nova.network.neutron [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Successfully created port: b3f5c844-2c18-4084-9948-08386b4b5276 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 631.192072] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407843, 'name': Rename_Task, 'duration_secs': 0.136788} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 631.192842] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 631.193898] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-8daa2e1e-b5c1-459f-8874-123f3dce71ee {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 631.200861] env[64020]: DEBUG nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 631.204059] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg ba6b15767ed2424e86113567f448d9ba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 631.217629] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 631.217629] env[64020]: value = "task-407844" [ 631.217629] env[64020]: _type = "Task" [ 631.217629] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 631.253716] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407844, 'name': PowerOnVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 631.265828] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba6b15767ed2424e86113567f448d9ba [ 631.326261] env[64020]: DEBUG nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 631.351609] env[64020]: DEBUG nova.virt.hardware [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 631.351848] env[64020]: DEBUG nova.virt.hardware [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 631.351995] env[64020]: DEBUG nova.virt.hardware [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 631.352226] env[64020]: DEBUG nova.virt.hardware [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 631.352384] env[64020]: DEBUG nova.virt.hardware [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 631.352524] env[64020]: DEBUG nova.virt.hardware [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 631.352727] env[64020]: DEBUG nova.virt.hardware [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 631.352878] env[64020]: DEBUG nova.virt.hardware [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 631.353034] env[64020]: DEBUG nova.virt.hardware [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 631.353184] env[64020]: DEBUG nova.virt.hardware [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 631.353354] env[64020]: DEBUG nova.virt.hardware [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 631.354588] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27990ed5-3270-4d03-b4bf-db019e6edc28 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 631.363734] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-064d99c7-8743-4593-a64c-5a8701c3aa89 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 631.508102] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.201s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 631.508772] env[64020]: ERROR nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b92962a4-4c4a-4ade-9372-98c32103ec91, please check neutron logs for more information. [ 631.508772] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Traceback (most recent call last): [ 631.508772] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 631.508772] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] self.driver.spawn(context, instance, image_meta, [ 631.508772] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 631.508772] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 631.508772] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 631.508772] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] vm_ref = self.build_virtual_machine(instance, [ 631.508772] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 631.508772] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] vif_infos = vmwarevif.get_vif_info(self._session, [ 631.508772] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] for vif in network_info: [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] return self._sync_wrapper(fn, *args, **kwargs) [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] self.wait() [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] self[:] = self._gt.wait() [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] return self._exit_event.wait() [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] result = hub.switch() [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 631.509189] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] return self.greenlet.switch() [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] result = function(*args, **kwargs) [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] return func(*args, **kwargs) [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] raise e [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] nwinfo = self.network_api.allocate_for_instance( [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] created_port_ids = self._update_ports_for_instance( [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] with excutils.save_and_reraise_exception(): [ 631.509574] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 631.509964] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] self.force_reraise() [ 631.509964] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 631.509964] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] raise self.value [ 631.509964] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 631.509964] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] updated_port = self._update_port( [ 631.509964] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 631.509964] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] _ensure_no_port_binding_failure(port) [ 631.509964] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 631.509964] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] raise exception.PortBindingFailed(port_id=port['id']) [ 631.509964] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] nova.exception.PortBindingFailed: Binding failed for port b92962a4-4c4a-4ade-9372-98c32103ec91, please check neutron logs for more information. [ 631.509964] env[64020]: ERROR nova.compute.manager [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] [ 631.510304] env[64020]: DEBUG nova.compute.utils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Binding failed for port b92962a4-4c4a-4ade-9372-98c32103ec91, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 631.510811] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 24.245s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 631.512308] env[64020]: INFO nova.compute.claims [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 631.514087] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 43b8410a76e94fcd9e007014931a557e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 631.515363] env[64020]: DEBUG nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Build of instance 5290c09b-d77c-4d27-b367-9de52f54d16a was re-scheduled: Binding failed for port b92962a4-4c4a-4ade-9372-98c32103ec91, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 631.515748] env[64020]: DEBUG nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 631.515969] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquiring lock "refresh_cache-5290c09b-d77c-4d27-b367-9de52f54d16a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 631.516254] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquired lock "refresh_cache-5290c09b-d77c-4d27-b367-9de52f54d16a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 631.516415] env[64020]: DEBUG nova.network.neutron [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 631.516901] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 6504e3c837e1487ba91a78b0b149c2a1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 631.524559] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6504e3c837e1487ba91a78b0b149c2a1 [ 631.552343] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43b8410a76e94fcd9e007014931a557e [ 631.731591] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407844, 'name': PowerOnVM_Task} progress is 66%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 631.732748] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 632.019920] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 178241fe76f949bca2d12444e40a9ddd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 632.035038] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 178241fe76f949bca2d12444e40a9ddd [ 632.089290] env[64020]: DEBUG nova.network.neutron [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 632.230024] env[64020]: DEBUG oslo_vmware.api [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407844, 'name': PowerOnVM_Task, 'duration_secs': 0.720585} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 632.230199] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 632.230419] env[64020]: DEBUG nova.compute.manager [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 632.231169] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a895ec7-634f-4573-a430-19ff6eba522f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 632.246200] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg b3601ffa11984c19ac1136e34ea49ce7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 632.294795] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3601ffa11984c19ac1136e34ea49ce7 [ 632.491278] env[64020]: DEBUG nova.network.neutron [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 632.491278] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 8767327f68e24662a9fd22af7ddaa048 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 632.499561] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8767327f68e24662a9fd22af7ddaa048 [ 632.759421] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 632.994505] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Releasing lock "refresh_cache-5290c09b-d77c-4d27-b367-9de52f54d16a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 632.994505] env[64020]: DEBUG nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 632.994757] env[64020]: DEBUG nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 632.994757] env[64020]: DEBUG nova.network.neutron [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 633.061302] env[64020]: DEBUG nova.network.neutron [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 633.061895] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 2052a9ce31034e219e5450cde63df945 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 633.073726] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2052a9ce31034e219e5450cde63df945 [ 633.119306] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1590435-09a2-4605-bcac-4df761c9a238 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.128526] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd166590-d9a5-411b-93aa-9a139ec4c82c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.169246] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00da865c-266e-4851-a743-141a35f9616a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.178166] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5cf4816c-70c7-4dc0-99ee-08a8906f5be9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.196118] env[64020]: DEBUG nova.compute.provider_tree [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 633.196638] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 63b8a09bde2c4319b6c7dce840e2f1eb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 633.206255] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 63b8a09bde2c4319b6c7dce840e2f1eb [ 633.564563] env[64020]: DEBUG nova.network.neutron [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 633.565078] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg aa175429a2bb4b67b6714f365d55e620 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 633.573468] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa175429a2bb4b67b6714f365d55e620 [ 633.699473] env[64020]: DEBUG nova.scheduler.client.report [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 633.701986] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 46a749035d3d44c696b19f0cc1c3358d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 633.714709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46a749035d3d44c696b19f0cc1c3358d [ 634.067265] env[64020]: INFO nova.compute.manager [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 5290c09b-d77c-4d27-b367-9de52f54d16a] Took 1.07 seconds to deallocate network for instance. [ 634.069318] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg ca063961f5554268a1bafa1aa9e7a57e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 634.116054] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca063961f5554268a1bafa1aa9e7a57e [ 634.204539] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.694s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 634.205059] env[64020]: DEBUG nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 634.207234] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 21878d2a309944feab020fa28c046d77 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 634.214679] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 19.299s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 634.214679] env[64020]: INFO nova.compute.claims [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 634.217139] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 236738b783b9492c8a7c0dabd790a0ae in queue reply_57893177120949e6a93cb88e15cd42b4 [ 634.275184] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21878d2a309944feab020fa28c046d77 [ 634.304114] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 236738b783b9492c8a7c0dabd790a0ae [ 634.575103] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg e3db7fe1a62b4de6948aac44f1005ba6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 634.628201] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Expecting reply to msg 9c327cdbe6684d3882234439c47224ce in queue reply_57893177120949e6a93cb88e15cd42b4 [ 634.637923] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3db7fe1a62b4de6948aac44f1005ba6 [ 634.647084] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c327cdbe6684d3882234439c47224ce [ 634.713303] env[64020]: DEBUG nova.compute.utils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 634.717217] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 353984ec32c74b94a204ae25fbf6ce18 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 634.717217] env[64020]: DEBUG nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 634.717217] env[64020]: DEBUG nova.network.neutron [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 634.727133] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 74d621cccb874c6399c78b69c5963510 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 634.728696] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 353984ec32c74b94a204ae25fbf6ce18 [ 634.741074] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74d621cccb874c6399c78b69c5963510 [ 634.950328] env[64020]: DEBUG nova.policy [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e53a328bbb6f41e88a2dafd9247170de', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7c874d59a6eb4aa2a9e2c9c575cc24aa', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 635.104726] env[64020]: INFO nova.scheduler.client.report [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Deleted allocations for instance 5290c09b-d77c-4d27-b367-9de52f54d16a [ 635.112151] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 13c5edb6427b49a6b22dfb68f12acde9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 635.129079] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 13c5edb6427b49a6b22dfb68f12acde9 [ 635.133257] env[64020]: INFO nova.compute.manager [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Rebuilding instance [ 635.189328] env[64020]: DEBUG nova.compute.manager [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 635.190325] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81d0346f-01bb-4127-bae2-90e8c0d84663 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.200049] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Expecting reply to msg 9a615b1d9fa546f8b06fc04260241380 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 635.218884] env[64020]: DEBUG nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 635.220942] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 36b9fd6e86074dd5b3a2a45312d5be44 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 635.268139] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9a615b1d9fa546f8b06fc04260241380 [ 635.268571] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36b9fd6e86074dd5b3a2a45312d5be44 [ 635.312693] env[64020]: ERROR nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b3f5c844-2c18-4084-9948-08386b4b5276, please check neutron logs for more information. [ 635.312693] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 635.312693] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 635.312693] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 635.312693] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 635.312693] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 635.312693] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 635.312693] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 635.312693] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 635.312693] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 635.312693] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 635.312693] env[64020]: ERROR nova.compute.manager raise self.value [ 635.312693] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 635.312693] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 635.312693] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 635.312693] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 635.313279] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 635.313279] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 635.313279] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b3f5c844-2c18-4084-9948-08386b4b5276, please check neutron logs for more information. [ 635.313279] env[64020]: ERROR nova.compute.manager [ 635.313279] env[64020]: Traceback (most recent call last): [ 635.313279] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 635.313279] env[64020]: listener.cb(fileno) [ 635.313279] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 635.313279] env[64020]: result = function(*args, **kwargs) [ 635.313279] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 635.313279] env[64020]: return func(*args, **kwargs) [ 635.313279] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 635.313279] env[64020]: raise e [ 635.313279] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 635.313279] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 635.313279] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 635.313279] env[64020]: created_port_ids = self._update_ports_for_instance( [ 635.313279] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 635.313279] env[64020]: with excutils.save_and_reraise_exception(): [ 635.313279] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 635.313279] env[64020]: self.force_reraise() [ 635.313279] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 635.313279] env[64020]: raise self.value [ 635.313279] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 635.313279] env[64020]: updated_port = self._update_port( [ 635.313279] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 635.313279] env[64020]: _ensure_no_port_binding_failure(port) [ 635.313279] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 635.313279] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 635.314257] env[64020]: nova.exception.PortBindingFailed: Binding failed for port b3f5c844-2c18-4084-9948-08386b4b5276, please check neutron logs for more information. [ 635.314257] env[64020]: Removing descriptor: 17 [ 635.314257] env[64020]: ERROR nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b3f5c844-2c18-4084-9948-08386b4b5276, please check neutron logs for more information. [ 635.314257] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Traceback (most recent call last): [ 635.314257] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 635.314257] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] yield resources [ 635.314257] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 635.314257] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] self.driver.spawn(context, instance, image_meta, [ 635.314257] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 635.314257] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] self._vmops.spawn(context, instance, image_meta, injected_files, [ 635.314257] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 635.314257] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] vm_ref = self.build_virtual_machine(instance, [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] vif_infos = vmwarevif.get_vif_info(self._session, [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] for vif in network_info: [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] return self._sync_wrapper(fn, *args, **kwargs) [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] self.wait() [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] self[:] = self._gt.wait() [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] return self._exit_event.wait() [ 635.314622] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] result = hub.switch() [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] return self.greenlet.switch() [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] result = function(*args, **kwargs) [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] return func(*args, **kwargs) [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] raise e [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] nwinfo = self.network_api.allocate_for_instance( [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 635.315071] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] created_port_ids = self._update_ports_for_instance( [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] with excutils.save_and_reraise_exception(): [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] self.force_reraise() [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] raise self.value [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] updated_port = self._update_port( [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] _ensure_no_port_binding_failure(port) [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 635.315603] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] raise exception.PortBindingFailed(port_id=port['id']) [ 635.316074] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] nova.exception.PortBindingFailed: Binding failed for port b3f5c844-2c18-4084-9948-08386b4b5276, please check neutron logs for more information. [ 635.316074] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] [ 635.316074] env[64020]: INFO nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Terminating instance [ 635.316074] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquiring lock "refresh_cache-65be7d88-0c12-4ad3-adc7-d089a1b91c28" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 635.316074] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquired lock "refresh_cache-65be7d88-0c12-4ad3-adc7-d089a1b91c28" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 635.316074] env[64020]: DEBUG nova.network.neutron [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 635.316400] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 339643fd4add482abbc46f45fcc80dd8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 635.326704] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 339643fd4add482abbc46f45fcc80dd8 [ 635.613933] env[64020]: DEBUG oslo_concurrency.lockutils [None req-731d4c57-27f8-4769-8523-1f3a8da5a9c0 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "5290c09b-d77c-4d27-b367-9de52f54d16a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 47.919s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 635.614523] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 07be2e2bca9a4e5b91d171da00ad075d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 635.629387] env[64020]: DEBUG nova.network.neutron [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Successfully created port: 5cc36a52-7d80-43b2-bc8b-0cb6be1a5090 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 635.631969] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 07be2e2bca9a4e5b91d171da00ad075d [ 635.647466] env[64020]: DEBUG nova.compute.manager [req-bdf3d59b-86cc-4941-8758-80e705270b10 req-5eef3eba-c7de-4d77-ab6d-b540ff4b93f2 service nova] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Received event network-changed-b3f5c844-2c18-4084-9948-08386b4b5276 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 635.647834] env[64020]: DEBUG nova.compute.manager [req-bdf3d59b-86cc-4941-8758-80e705270b10 req-5eef3eba-c7de-4d77-ab6d-b540ff4b93f2 service nova] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Refreshing instance network info cache due to event network-changed-b3f5c844-2c18-4084-9948-08386b4b5276. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 635.648150] env[64020]: DEBUG oslo_concurrency.lockutils [req-bdf3d59b-86cc-4941-8758-80e705270b10 req-5eef3eba-c7de-4d77-ab6d-b540ff4b93f2 service nova] Acquiring lock "refresh_cache-65be7d88-0c12-4ad3-adc7-d089a1b91c28" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 635.703697] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 635.703974] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-51084c89-3cd5-4047-aa0e-4c1d7f53daaa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.715454] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Waiting for the task: (returnval){ [ 635.715454] env[64020]: value = "task-407847" [ 635.715454] env[64020]: _type = "Task" [ 635.715454] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 635.727402] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 8e298330816241d0b2a5dd993b8a995a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 635.728646] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407847, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 635.730594] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d1227b8-d7cb-4b59-945d-673e802043d2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.740159] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-248f1284-42e5-415b-9aa4-1def8c9cc27a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.771854] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8e298330816241d0b2a5dd993b8a995a [ 635.773066] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28a5d59c-e386-4b52-b7a2-5edbae321d44 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.782057] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f20f0bd7-80c6-4de8-ab4f-c9ee01de2084 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.796597] env[64020]: DEBUG nova.compute.provider_tree [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 635.797202] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg a8ae758bfc374f62b26e857bfbf48fca in queue reply_57893177120949e6a93cb88e15cd42b4 [ 635.805637] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8ae758bfc374f62b26e857bfbf48fca [ 635.861521] env[64020]: DEBUG nova.network.neutron [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 636.030165] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Acquiring lock "47516f85-60d8-4931-a205-b2ec1f7aec83" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 636.035190] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Lock "47516f85-60d8-4931-a205-b2ec1f7aec83" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.005s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 636.088195] env[64020]: DEBUG nova.network.neutron [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 636.088195] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 69082a474abf4b3a84ac53568c8efd48 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 636.097889] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69082a474abf4b3a84ac53568c8efd48 [ 636.121558] env[64020]: DEBUG nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 636.121558] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg c7fbdce1385e43f8a9b38886971c0a82 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 636.166342] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c7fbdce1385e43f8a9b38886971c0a82 [ 636.226616] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407847, 'name': PowerOffVM_Task, 'duration_secs': 0.215229} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 636.226898] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 636.227085] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 636.228234] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf1c2a35-2471-41d0-afa8-372c394ebd4f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.232690] env[64020]: DEBUG nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 636.243952] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 636.244261] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-46735b59-02c1-4d52-99cb-863e3a63a448 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.262505] env[64020]: DEBUG nova.virt.hardware [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 636.262745] env[64020]: DEBUG nova.virt.hardware [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 636.262896] env[64020]: DEBUG nova.virt.hardware [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 636.263077] env[64020]: DEBUG nova.virt.hardware [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 636.263232] env[64020]: DEBUG nova.virt.hardware [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 636.263352] env[64020]: DEBUG nova.virt.hardware [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 636.263592] env[64020]: DEBUG nova.virt.hardware [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 636.263746] env[64020]: DEBUG nova.virt.hardware [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 636.263902] env[64020]: DEBUG nova.virt.hardware [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 636.264082] env[64020]: DEBUG nova.virt.hardware [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 636.264252] env[64020]: DEBUG nova.virt.hardware [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 636.265124] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15c4f7d6-8598-4c50-b408-f99a85b414e7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.274784] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-febd68fa-ad43-4fe0-a017-c49ca47e7234 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.284299] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 636.284559] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Deleting contents of the VM from datastore datastore2 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 636.284738] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Deleting the datastore file [datastore2] b3440b55-2469-46fa-ac2c-3e207bf530ec {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 636.284983] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-fba3b3b2-b546-47a7-889c-5d568dc0e34c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.301000] env[64020]: DEBUG nova.scheduler.client.report [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 636.303301] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg b918337d0a6b4ff195cf2f4f19d2ec68 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 636.308068] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Waiting for the task: (returnval){ [ 636.308068] env[64020]: value = "task-407850" [ 636.308068] env[64020]: _type = "Task" [ 636.308068] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 636.314909] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407850, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 636.315505] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b918337d0a6b4ff195cf2f4f19d2ec68 [ 636.590148] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Releasing lock "refresh_cache-65be7d88-0c12-4ad3-adc7-d089a1b91c28" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 636.590597] env[64020]: DEBUG nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 636.590805] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 636.591164] env[64020]: DEBUG oslo_concurrency.lockutils [req-bdf3d59b-86cc-4941-8758-80e705270b10 req-5eef3eba-c7de-4d77-ab6d-b540ff4b93f2 service nova] Acquired lock "refresh_cache-65be7d88-0c12-4ad3-adc7-d089a1b91c28" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 636.591347] env[64020]: DEBUG nova.network.neutron [req-bdf3d59b-86cc-4941-8758-80e705270b10 req-5eef3eba-c7de-4d77-ab6d-b540ff4b93f2 service nova] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Refreshing network info cache for port b3f5c844-2c18-4084-9948-08386b4b5276 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 636.591782] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-bdf3d59b-86cc-4941-8758-80e705270b10 req-5eef3eba-c7de-4d77-ab6d-b540ff4b93f2 service nova] Expecting reply to msg 933c9e468a2042179eb85a265eeae606 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 636.592620] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f6e0579b-bf46-4e44-9c32-09ce10693b28 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.602440] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6274524-78ff-4469-9237-3adda9fc797f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.612980] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 933c9e468a2042179eb85a265eeae606 [ 636.629000] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 65be7d88-0c12-4ad3-adc7-d089a1b91c28 could not be found. [ 636.629250] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 636.629422] env[64020]: INFO nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Took 0.04 seconds to destroy the instance on the hypervisor. [ 636.629675] env[64020]: DEBUG oslo.service.loopingcall [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 636.632114] env[64020]: DEBUG nova.compute.manager [-] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 636.632216] env[64020]: DEBUG nova.network.neutron [-] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 636.654380] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 636.657462] env[64020]: DEBUG nova.network.neutron [-] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 636.657462] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8e5716f5a3024f45859052a9f1a6bfc4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 636.662704] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8e5716f5a3024f45859052a9f1a6bfc4 [ 636.807551] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.597s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 636.809382] env[64020]: DEBUG nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 636.810536] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg fc8f5f85ea2642c289a6f90de67f5e9c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 636.812287] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 20.597s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 636.814163] env[64020]: INFO nova.compute.claims [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 636.815685] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 79c10bbf66e14342b98f092f000800be in queue reply_57893177120949e6a93cb88e15cd42b4 [ 636.826706] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407850, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.105081} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 636.826960] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 636.827144] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Deleted contents of the VM from datastore datastore2 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 636.827361] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 636.831481] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Expecting reply to msg a0b9fc1654504ff3afd62a0f135bb6bb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 636.865000] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 79c10bbf66e14342b98f092f000800be [ 636.869067] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc8f5f85ea2642c289a6f90de67f5e9c [ 636.902665] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a0b9fc1654504ff3afd62a0f135bb6bb [ 637.131211] env[64020]: DEBUG nova.network.neutron [req-bdf3d59b-86cc-4941-8758-80e705270b10 req-5eef3eba-c7de-4d77-ab6d-b540ff4b93f2 service nova] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 637.157666] env[64020]: DEBUG nova.network.neutron [-] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 637.157940] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 731bee99927b4458af40690894402000 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 637.166060] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 731bee99927b4458af40690894402000 [ 637.322489] env[64020]: DEBUG nova.compute.utils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 637.323641] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 4dca0deb539d4e829c0fd93ccd7e1959 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 637.325307] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 943b568e9c27475b9c7770ac6782a4a6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 637.326097] env[64020]: DEBUG nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 637.326240] env[64020]: DEBUG nova.network.neutron [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 637.333395] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 943b568e9c27475b9c7770ac6782a4a6 [ 637.339724] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Expecting reply to msg 9b1e27b428d04df4885ee44fc5712f5d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 637.343756] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4dca0deb539d4e829c0fd93ccd7e1959 [ 637.345450] env[64020]: DEBUG nova.network.neutron [req-bdf3d59b-86cc-4941-8758-80e705270b10 req-5eef3eba-c7de-4d77-ab6d-b540ff4b93f2 service nova] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 637.345823] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-bdf3d59b-86cc-4941-8758-80e705270b10 req-5eef3eba-c7de-4d77-ab6d-b540ff4b93f2 service nova] Expecting reply to msg a792977cd932446db5b9b9b61c158c43 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 637.358451] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a792977cd932446db5b9b9b61c158c43 [ 637.358451] env[64020]: ERROR nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 5cc36a52-7d80-43b2-bc8b-0cb6be1a5090, please check neutron logs for more information. [ 637.358451] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 637.358451] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 637.358451] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 637.358451] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 637.358451] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 637.358451] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 637.358451] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 637.358451] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 637.358451] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 637.358451] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 637.358451] env[64020]: ERROR nova.compute.manager raise self.value [ 637.358451] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 637.358451] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 637.358451] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 637.359038] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 637.359038] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 637.359038] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 637.359038] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 5cc36a52-7d80-43b2-bc8b-0cb6be1a5090, please check neutron logs for more information. [ 637.359038] env[64020]: ERROR nova.compute.manager [ 637.359038] env[64020]: Traceback (most recent call last): [ 637.359038] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 637.359038] env[64020]: listener.cb(fileno) [ 637.359038] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 637.359038] env[64020]: result = function(*args, **kwargs) [ 637.359038] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 637.359038] env[64020]: return func(*args, **kwargs) [ 637.359038] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 637.359038] env[64020]: raise e [ 637.359038] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 637.359038] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 637.359038] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 637.359038] env[64020]: created_port_ids = self._update_ports_for_instance( [ 637.359038] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 637.359038] env[64020]: with excutils.save_and_reraise_exception(): [ 637.359038] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 637.359038] env[64020]: self.force_reraise() [ 637.359038] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 637.359038] env[64020]: raise self.value [ 637.359038] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 637.359038] env[64020]: updated_port = self._update_port( [ 637.359038] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 637.359038] env[64020]: _ensure_no_port_binding_failure(port) [ 637.360132] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 637.360132] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 637.360132] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 5cc36a52-7d80-43b2-bc8b-0cb6be1a5090, please check neutron logs for more information. [ 637.360132] env[64020]: Removing descriptor: 16 [ 637.360132] env[64020]: ERROR nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 5cc36a52-7d80-43b2-bc8b-0cb6be1a5090, please check neutron logs for more information. [ 637.360132] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Traceback (most recent call last): [ 637.360132] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 637.360132] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] yield resources [ 637.360132] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 637.360132] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] self.driver.spawn(context, instance, image_meta, [ 637.360132] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 637.360132] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 637.360132] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] vm_ref = self.build_virtual_machine(instance, [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] vif_infos = vmwarevif.get_vif_info(self._session, [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] for vif in network_info: [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] return self._sync_wrapper(fn, *args, **kwargs) [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] self.wait() [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] self[:] = self._gt.wait() [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 637.360549] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] return self._exit_event.wait() [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] result = hub.switch() [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] return self.greenlet.switch() [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] result = function(*args, **kwargs) [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] return func(*args, **kwargs) [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] raise e [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] nwinfo = self.network_api.allocate_for_instance( [ 637.360954] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] created_port_ids = self._update_ports_for_instance( [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] with excutils.save_and_reraise_exception(): [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] self.force_reraise() [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] raise self.value [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] updated_port = self._update_port( [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] _ensure_no_port_binding_failure(port) [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 637.361400] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] raise exception.PortBindingFailed(port_id=port['id']) [ 637.361831] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] nova.exception.PortBindingFailed: Binding failed for port 5cc36a52-7d80-43b2-bc8b-0cb6be1a5090, please check neutron logs for more information. [ 637.361831] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] [ 637.361831] env[64020]: INFO nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Terminating instance [ 637.363520] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquiring lock "refresh_cache-e5b12149-0e4c-41b2-b726-f8a55452669a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 637.363647] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquired lock "refresh_cache-e5b12149-0e4c-41b2-b726-f8a55452669a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 637.363838] env[64020]: DEBUG nova.network.neutron [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 637.364973] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 0a047d93ef6d4e28a2972361944774ba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 637.396559] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a047d93ef6d4e28a2972361944774ba [ 637.410049] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b1e27b428d04df4885ee44fc5712f5d [ 637.637216] env[64020]: DEBUG nova.policy [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '556840b7a10d4f43973d8b76dfce3c5c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2caec29a321e4fb699c92f029795ee87', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 637.661268] env[64020]: INFO nova.compute.manager [-] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Took 1.03 seconds to deallocate network for instance. [ 637.669454] env[64020]: DEBUG nova.compute.claims [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 637.669627] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 637.812811] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8bd2105-42bd-400f-a3eb-90328597285b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.822277] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae034ca4-344f-4a79-ab10-ff75bf3c4db0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.827176] env[64020]: DEBUG nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 637.829047] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 653964101e924d918b032aa81c89c95e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 637.919970] env[64020]: DEBUG oslo_concurrency.lockutils [req-bdf3d59b-86cc-4941-8758-80e705270b10 req-5eef3eba-c7de-4d77-ab6d-b540ff4b93f2 service nova] Releasing lock "refresh_cache-65be7d88-0c12-4ad3-adc7-d089a1b91c28" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 637.922423] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 653964101e924d918b032aa81c89c95e [ 637.923648] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92131b2e-a484-4e69-8b8e-02c743bec75d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.933753] env[64020]: DEBUG nova.virt.hardware [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 637.934046] env[64020]: DEBUG nova.virt.hardware [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 637.934202] env[64020]: DEBUG nova.virt.hardware [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 637.934382] env[64020]: DEBUG nova.virt.hardware [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 637.934521] env[64020]: DEBUG nova.virt.hardware [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 637.934660] env[64020]: DEBUG nova.virt.hardware [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 637.934897] env[64020]: DEBUG nova.virt.hardware [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 637.935006] env[64020]: DEBUG nova.virt.hardware [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 637.935321] env[64020]: DEBUG nova.virt.hardware [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 637.935321] env[64020]: DEBUG nova.virt.hardware [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 637.935465] env[64020]: DEBUG nova.virt.hardware [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 637.936808] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c130d3e4-dc73-4c62-91b3-f77dfe3ca75e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.943110] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b48447e3-8137-4c8c-a4f2-3b41a07910ee {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.957041] env[64020]: DEBUG nova.compute.provider_tree [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 637.957041] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 86473d7b91fb468d80e732d2ccd1ceab in queue reply_57893177120949e6a93cb88e15cd42b4 [ 637.964647] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7471d77-2f57-4744-9674-f6381d5e27c0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.979416] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86473d7b91fb468d80e732d2ccd1ceab [ 637.980240] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 637.986125] env[64020]: DEBUG oslo.service.loopingcall [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 637.986717] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 637.986945] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-d2aeb165-115f-4f96-8f87-fc1ed3268c07 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.000803] env[64020]: DEBUG nova.network.neutron [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 638.005985] env[64020]: DEBUG nova.compute.manager [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Received event network-vif-deleted-b3f5c844-2c18-4084-9948-08386b4b5276 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 638.006199] env[64020]: DEBUG nova.compute.manager [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Received event network-changed-5cc36a52-7d80-43b2-bc8b-0cb6be1a5090 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 638.006383] env[64020]: DEBUG nova.compute.manager [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Refreshing instance network info cache due to event network-changed-5cc36a52-7d80-43b2-bc8b-0cb6be1a5090. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 638.006590] env[64020]: DEBUG oslo_concurrency.lockutils [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] Acquiring lock "refresh_cache-e5b12149-0e4c-41b2-b726-f8a55452669a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 638.016072] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 638.016072] env[64020]: value = "task-407852" [ 638.016072] env[64020]: _type = "Task" [ 638.016072] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 638.035777] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407852, 'name': CreateVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 638.347083] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 6231199812474fe6b54694f5abaacdba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 638.407945] env[64020]: DEBUG nova.network.neutron [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 638.408565] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 8fe928ed897b41c6ba3aa7728520d594 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 638.421545] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6231199812474fe6b54694f5abaacdba [ 638.437964] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8fe928ed897b41c6ba3aa7728520d594 [ 638.438551] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquiring lock "2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 638.438751] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 638.466085] env[64020]: DEBUG nova.scheduler.client.report [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 638.468574] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg c06b1ecf212a4fff8aa75a60dbfce3dd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 638.480540] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c06b1ecf212a4fff8aa75a60dbfce3dd [ 638.526880] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407852, 'name': CreateVM_Task, 'duration_secs': 0.345379} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 638.527040] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 638.527448] env[64020]: DEBUG oslo_concurrency.lockutils [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 638.527578] env[64020]: DEBUG oslo_concurrency.lockutils [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 638.527905] env[64020]: DEBUG oslo_concurrency.lockutils [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Acquired external semaphore "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 638.528164] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-1ea76d3c-1ea8-47c7-b4d8-1007436158b6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.533505] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Waiting for the task: (returnval){ [ 638.533505] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52fb30e0-5e26-909e-0440-1e57b78610fe" [ 638.533505] env[64020]: _type = "Task" [ 638.533505] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 638.543037] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52fb30e0-5e26-909e-0440-1e57b78610fe, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 638.842727] env[64020]: DEBUG nova.network.neutron [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Successfully created port: 1382c442-7ac3-48fc-b680-81c4ea9727f2 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 638.849930] env[64020]: DEBUG nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 638.883356] env[64020]: DEBUG nova.virt.hardware [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 638.883575] env[64020]: DEBUG nova.virt.hardware [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 638.883729] env[64020]: DEBUG nova.virt.hardware [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 638.883904] env[64020]: DEBUG nova.virt.hardware [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 638.884058] env[64020]: DEBUG nova.virt.hardware [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 638.884207] env[64020]: DEBUG nova.virt.hardware [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 638.884406] env[64020]: DEBUG nova.virt.hardware [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 638.884559] env[64020]: DEBUG nova.virt.hardware [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 638.884718] env[64020]: DEBUG nova.virt.hardware [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 638.885006] env[64020]: DEBUG nova.virt.hardware [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 638.885089] env[64020]: DEBUG nova.virt.hardware [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 638.885851] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79bbb305-c09d-4b53-bf5f-1064b20f5872 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.897047] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ad2d423-93f4-456b-b8b8-f58dc8c01f53 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.914420] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Releasing lock "refresh_cache-e5b12149-0e4c-41b2-b726-f8a55452669a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 638.915157] env[64020]: DEBUG nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 638.915363] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 638.915883] env[64020]: DEBUG oslo_concurrency.lockutils [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] Acquired lock "refresh_cache-e5b12149-0e4c-41b2-b726-f8a55452669a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 638.916130] env[64020]: DEBUG nova.network.neutron [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Refreshing network info cache for port 5cc36a52-7d80-43b2-bc8b-0cb6be1a5090 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 638.916570] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] Expecting reply to msg 7dedfef3537846708bfe4319cef9f1c2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 638.917388] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bfdc0d6d-a0a7-4e09-a832-56d6103af5be {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.930151] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7dedfef3537846708bfe4319cef9f1c2 [ 638.940462] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb86d0dc-b8fd-445e-aa19-956ffeb8635f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.970509] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e5b12149-0e4c-41b2-b726-f8a55452669a could not be found. [ 638.970785] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 638.970988] env[64020]: INFO nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Took 0.06 seconds to destroy the instance on the hypervisor. [ 638.971278] env[64020]: DEBUG oslo.service.loopingcall [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 638.972658] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.160s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 638.973227] env[64020]: DEBUG nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 638.974962] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 364e8e1b1d684b44bba1d44de61ed3df in queue reply_57893177120949e6a93cb88e15cd42b4 [ 638.976169] env[64020]: DEBUG nova.compute.manager [-] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 638.976274] env[64020]: DEBUG nova.network.neutron [-] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 638.978257] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.825s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 638.980087] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 1caa9741e9384ddea2537e618eb8ebf3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 639.017965] env[64020]: DEBUG nova.network.neutron [-] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 639.018502] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 82026725ae5a455fa104b432673eb91d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 639.038283] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82026725ae5a455fa104b432673eb91d [ 639.045658] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52fb30e0-5e26-909e-0440-1e57b78610fe, 'name': SearchDatastore_Task, 'duration_secs': 0.009153} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 639.045951] env[64020]: DEBUG oslo_concurrency.lockutils [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 639.046173] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 639.046401] env[64020]: DEBUG oslo_concurrency.lockutils [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 639.046601] env[64020]: DEBUG oslo_concurrency.lockutils [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 639.046708] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 639.046955] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-05ea8965-a31b-4c00-9ede-5bbba1fa84e0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.055596] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1caa9741e9384ddea2537e618eb8ebf3 [ 639.059638] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Created directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 639.060130] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Folder [datastore1] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 639.061191] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 364e8e1b1d684b44bba1d44de61ed3df [ 639.061524] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c15afddb-ad16-40e2-9123-ecd72c67a6c5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.069026] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Waiting for the task: (returnval){ [ 639.069026] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52cb85ef-80d3-f5a8-888f-cf7d34f80c19" [ 639.069026] env[64020]: _type = "Task" [ 639.069026] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 639.082152] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52cb85ef-80d3-f5a8-888f-cf7d34f80c19, 'name': SearchDatastore_Task, 'duration_secs': 0.009641} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 639.083838] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8872af1c-a1c8-4b8f-aa41-fc00763cb0d5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.090540] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Waiting for the task: (returnval){ [ 639.090540] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52be0317-4776-f574-66e2-641e5f07f0ff" [ 639.090540] env[64020]: _type = "Task" [ 639.090540] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 639.101370] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52be0317-4776-f574-66e2-641e5f07f0ff, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 639.483933] env[64020]: DEBUG nova.compute.utils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 639.483933] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 1189ab796da9459d80394624e6139e9d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 639.483933] env[64020]: DEBUG nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 639.483933] env[64020]: DEBUG nova.network.neutron [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 639.490324] env[64020]: DEBUG nova.network.neutron [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 639.502752] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1189ab796da9459d80394624e6139e9d [ 639.522580] env[64020]: DEBUG nova.network.neutron [-] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 639.523159] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5226904077634816a244b7250e084c9b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 639.531885] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5226904077634816a244b7250e084c9b [ 639.604806] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52be0317-4776-f574-66e2-641e5f07f0ff, 'name': SearchDatastore_Task, 'duration_secs': 0.009896} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 639.605063] env[64020]: DEBUG oslo_concurrency.lockutils [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 639.605448] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Copying Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 639.605804] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-d450fab2-dcbb-4d54-97c1-550c01886e64 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.615788] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Waiting for the task: (returnval){ [ 639.615788] env[64020]: value = "task-407853" [ 639.615788] env[64020]: _type = "Task" [ 639.615788] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 639.624983] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407853, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 639.712581] env[64020]: DEBUG nova.policy [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2b175c17992b45f29fc1dd1bae5de8d4', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5dce267ed99746b39be0e2935c620dab', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 639.807152] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Acquiring lock "f873c026-0fa1-4176-a30f-207767f87410" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 639.807152] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Lock "f873c026-0fa1-4176-a30f-207767f87410" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 639.902005] env[64020]: DEBUG nova.network.neutron [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 639.902005] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] Expecting reply to msg f0940c2335254baabdef398c1df63bf2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 639.921153] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f0940c2335254baabdef398c1df63bf2 [ 639.985178] env[64020]: DEBUG nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 639.986849] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 5f734b468a1349538bb43a4e94562681 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 640.029174] env[64020]: INFO nova.compute.manager [-] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Took 1.05 seconds to deallocate network for instance. [ 640.034071] env[64020]: DEBUG nova.compute.claims [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 640.034299] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 640.056178] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5f734b468a1349538bb43a4e94562681 [ 640.105573] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc7785fe-7ce1-4801-a662-c63ed498a3a4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.117392] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0fa296f-fbd0-45e2-b6ef-1b176ecebc6f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.134595] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407853, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.467231} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 640.162122] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Copied Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 640.162511] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 640.163056] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-e74e2288-613b-4323-ab9e-b1d14ee30c3c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.166255] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32380014-cdda-4d27-bf49-573dbe25b5b3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.178775] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-591f2cbb-4721-4fb7-b24b-949a3dd12c59 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.185579] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Waiting for the task: (returnval){ [ 640.185579] env[64020]: value = "task-407855" [ 640.185579] env[64020]: _type = "Task" [ 640.185579] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 640.203479] env[64020]: DEBUG nova.compute.provider_tree [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 640.204017] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 9c724ae2a6a741b5a1e0f65e9ab25887 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 640.219741] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407855, 'name': ExtendVirtualDisk_Task} progress is 50%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 640.224960] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c724ae2a6a741b5a1e0f65e9ab25887 [ 640.411003] env[64020]: DEBUG oslo_concurrency.lockutils [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] Releasing lock "refresh_cache-e5b12149-0e4c-41b2-b726-f8a55452669a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 640.411198] env[64020]: DEBUG nova.compute.manager [req-afc55352-6314-4673-879b-3504afd91044 req-e231d4ba-6fbf-4b73-bd2f-18c5d3848586 service nova] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Received event network-vif-deleted-5cc36a52-7d80-43b2-bc8b-0cb6be1a5090 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 640.463587] env[64020]: DEBUG nova.network.neutron [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Successfully created port: 860ce71a-5c1b-48d2-8d62-3545e5714ce4 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 640.493470] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg b8573a8da60248eebd4dd902300367e8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 640.513489] env[64020]: ERROR nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1382c442-7ac3-48fc-b680-81c4ea9727f2, please check neutron logs for more information. [ 640.513489] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 640.513489] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 640.513489] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 640.513489] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 640.513489] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 640.513489] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 640.513489] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 640.513489] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 640.513489] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 640.513489] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 640.513489] env[64020]: ERROR nova.compute.manager raise self.value [ 640.513489] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 640.513489] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 640.513489] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 640.513489] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 640.514286] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 640.514286] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 640.514286] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1382c442-7ac3-48fc-b680-81c4ea9727f2, please check neutron logs for more information. [ 640.514286] env[64020]: ERROR nova.compute.manager [ 640.514286] env[64020]: Traceback (most recent call last): [ 640.514286] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 640.514286] env[64020]: listener.cb(fileno) [ 640.514286] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 640.514286] env[64020]: result = function(*args, **kwargs) [ 640.514286] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 640.514286] env[64020]: return func(*args, **kwargs) [ 640.514286] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 640.514286] env[64020]: raise e [ 640.514286] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 640.514286] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 640.514286] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 640.514286] env[64020]: created_port_ids = self._update_ports_for_instance( [ 640.514286] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 640.514286] env[64020]: with excutils.save_and_reraise_exception(): [ 640.514286] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 640.514286] env[64020]: self.force_reraise() [ 640.514286] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 640.514286] env[64020]: raise self.value [ 640.514286] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 640.514286] env[64020]: updated_port = self._update_port( [ 640.514286] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 640.514286] env[64020]: _ensure_no_port_binding_failure(port) [ 640.514286] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 640.514286] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 640.515738] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 1382c442-7ac3-48fc-b680-81c4ea9727f2, please check neutron logs for more information. [ 640.515738] env[64020]: Removing descriptor: 18 [ 640.515738] env[64020]: ERROR nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1382c442-7ac3-48fc-b680-81c4ea9727f2, please check neutron logs for more information. [ 640.515738] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Traceback (most recent call last): [ 640.515738] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 640.515738] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] yield resources [ 640.515738] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 640.515738] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] self.driver.spawn(context, instance, image_meta, [ 640.515738] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 640.515738] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 640.515738] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 640.515738] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] vm_ref = self.build_virtual_machine(instance, [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] vif_infos = vmwarevif.get_vif_info(self._session, [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] for vif in network_info: [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] return self._sync_wrapper(fn, *args, **kwargs) [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] self.wait() [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] self[:] = self._gt.wait() [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] return self._exit_event.wait() [ 640.516323] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] result = hub.switch() [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] return self.greenlet.switch() [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] result = function(*args, **kwargs) [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] return func(*args, **kwargs) [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] raise e [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] nwinfo = self.network_api.allocate_for_instance( [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 640.517080] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] created_port_ids = self._update_ports_for_instance( [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] with excutils.save_and_reraise_exception(): [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] self.force_reraise() [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] raise self.value [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] updated_port = self._update_port( [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] _ensure_no_port_binding_failure(port) [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 640.517715] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] raise exception.PortBindingFailed(port_id=port['id']) [ 640.522418] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] nova.exception.PortBindingFailed: Binding failed for port 1382c442-7ac3-48fc-b680-81c4ea9727f2, please check neutron logs for more information. [ 640.522418] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] [ 640.522418] env[64020]: INFO nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Terminating instance [ 640.522418] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Acquiring lock "refresh_cache-a98ba289-b065-4cef-a7bd-7b7cd07b141d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 640.522418] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Acquired lock "refresh_cache-a98ba289-b065-4cef-a7bd-7b7cd07b141d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 640.522418] env[64020]: DEBUG nova.network.neutron [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 640.522975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 9b3e0b4c68d7483b82a94a28dc009139 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 640.524613] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b3e0b4c68d7483b82a94a28dc009139 [ 640.530389] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8573a8da60248eebd4dd902300367e8 [ 640.696143] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407855, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.074625} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 640.696397] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 640.697190] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac611fba-ad49-4e5b-b92b-13ba1c8f0d08 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.718524] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Reconfiguring VM instance instance-00000009 to attach disk [datastore1] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 640.719499] env[64020]: DEBUG nova.scheduler.client.report [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 640.721932] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 95616e61005b40b09326d91a74035919 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 640.727776] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-f49312ca-2bb7-46b0-ac4d-dacdd9d6792c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.740797] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95616e61005b40b09326d91a74035919 [ 640.749207] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Waiting for the task: (returnval){ [ 640.749207] env[64020]: value = "task-407856" [ 640.749207] env[64020]: _type = "Task" [ 640.749207] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 640.759732] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407856, 'name': ReconfigVM_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 640.996935] env[64020]: DEBUG nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 641.031786] env[64020]: DEBUG nova.virt.hardware [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 641.032181] env[64020]: DEBUG nova.virt.hardware [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 641.032431] env[64020]: DEBUG nova.virt.hardware [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 641.032797] env[64020]: DEBUG nova.virt.hardware [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 641.032876] env[64020]: DEBUG nova.virt.hardware [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 641.033082] env[64020]: DEBUG nova.virt.hardware [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 641.033389] env[64020]: DEBUG nova.virt.hardware [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 641.033720] env[64020]: DEBUG nova.virt.hardware [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 641.033972] env[64020]: DEBUG nova.virt.hardware [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 641.034247] env[64020]: DEBUG nova.virt.hardware [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 641.034719] env[64020]: DEBUG nova.virt.hardware [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 641.035627] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fbf52db8-5b8a-4cd8-9e92-4d5f1e61cffa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.050923] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3d192a0-87fc-4892-9cbf-b95cfb86dfa9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.064386] env[64020]: DEBUG nova.network.neutron [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 641.225535] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.247s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 641.226167] env[64020]: ERROR nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 57a9791c-3911-480c-bb77-c7950afb78c3, please check neutron logs for more information. [ 641.226167] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Traceback (most recent call last): [ 641.226167] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 641.226167] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] self.driver.spawn(context, instance, image_meta, [ 641.226167] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 641.226167] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 641.226167] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 641.226167] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] vm_ref = self.build_virtual_machine(instance, [ 641.226167] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 641.226167] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] vif_infos = vmwarevif.get_vif_info(self._session, [ 641.226167] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] for vif in network_info: [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] return self._sync_wrapper(fn, *args, **kwargs) [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] self.wait() [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] self[:] = self._gt.wait() [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] return self._exit_event.wait() [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] result = hub.switch() [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 641.226741] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] return self.greenlet.switch() [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] result = function(*args, **kwargs) [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] return func(*args, **kwargs) [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] raise e [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] nwinfo = self.network_api.allocate_for_instance( [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] created_port_ids = self._update_ports_for_instance( [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] with excutils.save_and_reraise_exception(): [ 641.227134] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 641.227526] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] self.force_reraise() [ 641.227526] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 641.227526] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] raise self.value [ 641.227526] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 641.227526] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] updated_port = self._update_port( [ 641.227526] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 641.227526] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] _ensure_no_port_binding_failure(port) [ 641.227526] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 641.227526] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] raise exception.PortBindingFailed(port_id=port['id']) [ 641.227526] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] nova.exception.PortBindingFailed: Binding failed for port 57a9791c-3911-480c-bb77-c7950afb78c3, please check neutron logs for more information. [ 641.227526] env[64020]: ERROR nova.compute.manager [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] [ 641.228830] env[64020]: DEBUG nova.compute.utils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Binding failed for port 57a9791c-3911-480c-bb77-c7950afb78c3, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 641.228830] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.662s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 641.230195] env[64020]: INFO nova.compute.claims [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 641.231878] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 82469eb4ec444a7bac65e3573c6f1617 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 641.233603] env[64020]: DEBUG nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Build of instance 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c was re-scheduled: Binding failed for port 57a9791c-3911-480c-bb77-c7950afb78c3, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 641.234055] env[64020]: DEBUG nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 641.234281] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Acquiring lock "refresh_cache-1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 641.234417] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Acquired lock "refresh_cache-1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 641.234570] env[64020]: DEBUG nova.network.neutron [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 641.235223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 2d95003a5c7841d0a1c3d818e06c637f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 641.243562] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d95003a5c7841d0a1c3d818e06c637f [ 641.267994] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407856, 'name': ReconfigVM_Task} progress is 14%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 641.280847] env[64020]: DEBUG nova.network.neutron [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 641.281716] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg cfb693aa44344c76878fbbcd1e1d0be2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 641.282916] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82469eb4ec444a7bac65e3573c6f1617 [ 641.289888] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfb693aa44344c76878fbbcd1e1d0be2 [ 641.634350] env[64020]: DEBUG nova.compute.manager [req-2678b720-1bce-4b22-958b-15ead1c9b1a2 req-11ceb8a2-bf72-41d3-96c6-894f4f9a812e service nova] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Received event network-changed-1382c442-7ac3-48fc-b680-81c4ea9727f2 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 641.634645] env[64020]: DEBUG nova.compute.manager [req-2678b720-1bce-4b22-958b-15ead1c9b1a2 req-11ceb8a2-bf72-41d3-96c6-894f4f9a812e service nova] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Refreshing instance network info cache due to event network-changed-1382c442-7ac3-48fc-b680-81c4ea9727f2. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 641.634714] env[64020]: DEBUG oslo_concurrency.lockutils [req-2678b720-1bce-4b22-958b-15ead1c9b1a2 req-11ceb8a2-bf72-41d3-96c6-894f4f9a812e service nova] Acquiring lock "refresh_cache-a98ba289-b065-4cef-a7bd-7b7cd07b141d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 641.738629] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 84b0d7e3678f4d468fa01da69ac97d67 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 641.746529] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 84b0d7e3678f4d468fa01da69ac97d67 [ 641.760340] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407856, 'name': ReconfigVM_Task, 'duration_secs': 0.831855} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 641.760543] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Reconfigured VM instance instance-00000009 to attach disk [datastore1] b3440b55-2469-46fa-ac2c-3e207bf530ec/b3440b55-2469-46fa-ac2c-3e207bf530ec.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 641.761234] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-250db0fb-cda8-4a76-93fb-f834b9188d4a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.769541] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Waiting for the task: (returnval){ [ 641.769541] env[64020]: value = "task-407857" [ 641.769541] env[64020]: _type = "Task" [ 641.769541] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 641.779176] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407857, 'name': Rename_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 641.783999] env[64020]: DEBUG nova.network.neutron [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 641.786187] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Releasing lock "refresh_cache-a98ba289-b065-4cef-a7bd-7b7cd07b141d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 641.786618] env[64020]: DEBUG nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 641.786809] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 641.787105] env[64020]: DEBUG oslo_concurrency.lockutils [req-2678b720-1bce-4b22-958b-15ead1c9b1a2 req-11ceb8a2-bf72-41d3-96c6-894f4f9a812e service nova] Acquired lock "refresh_cache-a98ba289-b065-4cef-a7bd-7b7cd07b141d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 641.787462] env[64020]: DEBUG nova.network.neutron [req-2678b720-1bce-4b22-958b-15ead1c9b1a2 req-11ceb8a2-bf72-41d3-96c6-894f4f9a812e service nova] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Refreshing network info cache for port 1382c442-7ac3-48fc-b680-81c4ea9727f2 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 641.787920] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-2678b720-1bce-4b22-958b-15ead1c9b1a2 req-11ceb8a2-bf72-41d3-96c6-894f4f9a812e service nova] Expecting reply to msg 7551685a1d694aebb3defa428e9e5eff in queue reply_57893177120949e6a93cb88e15cd42b4 [ 641.788704] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7bb0afb1-5a89-4c69-9b64-cb27ac1e94b7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.799669] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-699c9709-63bc-457c-b18f-1d6f295282d1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.825044] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7551685a1d694aebb3defa428e9e5eff [ 641.834741] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a98ba289-b065-4cef-a7bd-7b7cd07b141d could not be found. [ 641.834968] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 641.835149] env[64020]: INFO nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Took 0.05 seconds to destroy the instance on the hypervisor. [ 641.835537] env[64020]: DEBUG oslo.service.loopingcall [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 641.835704] env[64020]: DEBUG nova.compute.manager [-] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 641.835796] env[64020]: DEBUG nova.network.neutron [-] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 641.888658] env[64020]: DEBUG nova.network.neutron [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 641.889267] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg d7d87ff5e79240f08e365d8f72613139 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 641.895065] env[64020]: DEBUG nova.network.neutron [-] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 641.895590] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7231869850354f3ab095d6e992365b18 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 641.901142] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7d87ff5e79240f08e365d8f72613139 [ 641.903936] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7231869850354f3ab095d6e992365b18 [ 642.023576] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Acquiring lock "052ae9f0-bb03-4747-b8f9-708e4a7c147c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 642.023841] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Lock "052ae9f0-bb03-4747-b8f9-708e4a7c147c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.290023] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407857, 'name': Rename_Task, 'duration_secs': 0.173802} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 642.290311] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 642.290566] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-6111cc02-1c3a-4796-987c-f7798a5fed2b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.311246] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Waiting for the task: (returnval){ [ 642.311246] env[64020]: value = "task-407858" [ 642.311246] env[64020]: _type = "Task" [ 642.311246] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 642.323358] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407858, 'name': PowerOnVM_Task} progress is 33%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 642.372396] env[64020]: DEBUG nova.network.neutron [req-2678b720-1bce-4b22-958b-15ead1c9b1a2 req-11ceb8a2-bf72-41d3-96c6-894f4f9a812e service nova] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 642.391678] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Releasing lock "refresh_cache-1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 642.391909] env[64020]: DEBUG nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 642.392088] env[64020]: DEBUG nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 642.392260] env[64020]: DEBUG nova.network.neutron [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 642.398502] env[64020]: DEBUG nova.network.neutron [-] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 642.400433] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 067f8b3a2b7a45bea7e199054ac550a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 642.409448] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 067f8b3a2b7a45bea7e199054ac550a2 [ 642.423172] env[64020]: DEBUG nova.network.neutron [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 642.424898] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 67cb38d607944c6f9306f4857cffce6b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 642.440408] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 67cb38d607944c6f9306f4857cffce6b [ 642.712078] env[64020]: DEBUG nova.network.neutron [req-2678b720-1bce-4b22-958b-15ead1c9b1a2 req-11ceb8a2-bf72-41d3-96c6-894f4f9a812e service nova] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 642.712897] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-2678b720-1bce-4b22-958b-15ead1c9b1a2 req-11ceb8a2-bf72-41d3-96c6-894f4f9a812e service nova] Expecting reply to msg 6716f5e9bea94d53b2b0176817a03112 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 642.726148] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6716f5e9bea94d53b2b0176817a03112 [ 642.823433] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-326d23a6-01ef-45f7-b1a6-b0936dfcbb18 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.835997] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-212effbe-5e65-4f5c-b4a8-18e689d8c881 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.841832] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407858, 'name': PowerOnVM_Task} progress is 100%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 642.878552] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-108e4f91-15fc-41eb-8c51-b73b5f73896f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.887871] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1ac81c9-b03c-4978-8a8f-228eee27e635 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.904901] env[64020]: INFO nova.compute.manager [-] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Took 1.07 seconds to deallocate network for instance. [ 642.905614] env[64020]: DEBUG nova.compute.provider_tree [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 642.906138] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg e4c01d4ae4a643e0be8d3c6c1452ba42 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 642.912028] env[64020]: DEBUG nova.compute.claims [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 642.912028] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 642.917588] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e4c01d4ae4a643e0be8d3c6c1452ba42 [ 642.927390] env[64020]: DEBUG nova.network.neutron [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 642.927990] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 54ff240239a1485ea41cf7b4071ddb3c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 642.937590] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54ff240239a1485ea41cf7b4071ddb3c [ 643.154258] env[64020]: ERROR nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 860ce71a-5c1b-48d2-8d62-3545e5714ce4, please check neutron logs for more information. [ 643.154258] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 643.154258] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 643.154258] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 643.154258] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 643.154258] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 643.154258] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 643.154258] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 643.154258] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 643.154258] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 643.154258] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 643.154258] env[64020]: ERROR nova.compute.manager raise self.value [ 643.154258] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 643.154258] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 643.154258] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 643.154258] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 643.154870] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 643.154870] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 643.154870] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 860ce71a-5c1b-48d2-8d62-3545e5714ce4, please check neutron logs for more information. [ 643.154870] env[64020]: ERROR nova.compute.manager [ 643.154870] env[64020]: Traceback (most recent call last): [ 643.154870] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 643.154870] env[64020]: listener.cb(fileno) [ 643.154870] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 643.154870] env[64020]: result = function(*args, **kwargs) [ 643.154870] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 643.154870] env[64020]: return func(*args, **kwargs) [ 643.154870] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 643.154870] env[64020]: raise e [ 643.154870] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 643.154870] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 643.154870] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 643.154870] env[64020]: created_port_ids = self._update_ports_for_instance( [ 643.154870] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 643.154870] env[64020]: with excutils.save_and_reraise_exception(): [ 643.154870] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 643.154870] env[64020]: self.force_reraise() [ 643.154870] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 643.154870] env[64020]: raise self.value [ 643.154870] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 643.154870] env[64020]: updated_port = self._update_port( [ 643.154870] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 643.154870] env[64020]: _ensure_no_port_binding_failure(port) [ 643.154870] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 643.154870] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 643.155950] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 860ce71a-5c1b-48d2-8d62-3545e5714ce4, please check neutron logs for more information. [ 643.155950] env[64020]: Removing descriptor: 17 [ 643.155950] env[64020]: ERROR nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 860ce71a-5c1b-48d2-8d62-3545e5714ce4, please check neutron logs for more information. [ 643.155950] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Traceback (most recent call last): [ 643.155950] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 643.155950] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] yield resources [ 643.155950] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 643.155950] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] self.driver.spawn(context, instance, image_meta, [ 643.155950] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 643.155950] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 643.155950] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 643.155950] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] vm_ref = self.build_virtual_machine(instance, [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] vif_infos = vmwarevif.get_vif_info(self._session, [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] for vif in network_info: [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] return self._sync_wrapper(fn, *args, **kwargs) [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] self.wait() [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] self[:] = self._gt.wait() [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] return self._exit_event.wait() [ 643.156397] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] result = hub.switch() [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] return self.greenlet.switch() [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] result = function(*args, **kwargs) [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] return func(*args, **kwargs) [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] raise e [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] nwinfo = self.network_api.allocate_for_instance( [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 643.156854] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] created_port_ids = self._update_ports_for_instance( [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] with excutils.save_and_reraise_exception(): [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] self.force_reraise() [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] raise self.value [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] updated_port = self._update_port( [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] _ensure_no_port_binding_failure(port) [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 643.157259] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] raise exception.PortBindingFailed(port_id=port['id']) [ 643.157687] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] nova.exception.PortBindingFailed: Binding failed for port 860ce71a-5c1b-48d2-8d62-3545e5714ce4, please check neutron logs for more information. [ 643.157687] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] [ 643.157687] env[64020]: INFO nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Terminating instance [ 643.157687] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquiring lock "refresh_cache-ae57ce0e-46b5-4490-af3f-4b2537bf316a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 643.157842] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquired lock "refresh_cache-ae57ce0e-46b5-4490-af3f-4b2537bf316a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 643.157884] env[64020]: DEBUG nova.network.neutron [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 643.158260] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg fbdeb83d9c3c4f95a4326e7f7ebedb16 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 643.164999] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fbdeb83d9c3c4f95a4326e7f7ebedb16 [ 643.223596] env[64020]: DEBUG oslo_concurrency.lockutils [req-2678b720-1bce-4b22-958b-15ead1c9b1a2 req-11ceb8a2-bf72-41d3-96c6-894f4f9a812e service nova] Releasing lock "refresh_cache-a98ba289-b065-4cef-a7bd-7b7cd07b141d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 643.329337] env[64020]: DEBUG oslo_vmware.api [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Task: {'id': task-407858, 'name': PowerOnVM_Task, 'duration_secs': 0.53768} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 643.329337] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 643.329337] env[64020]: DEBUG nova.compute.manager [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 643.329670] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3aa7521c-6c6b-43e1-b45a-91f26a0789a9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.339459] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Expecting reply to msg 22f6e87558434bac97447b5d3b1a83ed in queue reply_57893177120949e6a93cb88e15cd42b4 [ 643.385878] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 22f6e87558434bac97447b5d3b1a83ed [ 643.410331] env[64020]: DEBUG nova.scheduler.client.report [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 643.413077] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 16dba56e670045b1a7dc44739320efdd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 643.425493] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16dba56e670045b1a7dc44739320efdd [ 643.436430] env[64020]: INFO nova.compute.manager [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] [instance: 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c] Took 1.04 seconds to deallocate network for instance. [ 643.438063] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 3237dc96b2584b05890b7808549fc8fb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 643.477631] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3237dc96b2584b05890b7808549fc8fb [ 643.523474] env[64020]: DEBUG nova.compute.manager [req-3a17deca-181a-4620-b774-0b2fcf1caf92 req-0665c027-dff0-46b4-9659-1773e1cdcf2e service nova] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Received event network-changed-860ce71a-5c1b-48d2-8d62-3545e5714ce4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 643.523657] env[64020]: DEBUG nova.compute.manager [req-3a17deca-181a-4620-b774-0b2fcf1caf92 req-0665c027-dff0-46b4-9659-1773e1cdcf2e service nova] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Refreshing instance network info cache due to event network-changed-860ce71a-5c1b-48d2-8d62-3545e5714ce4. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 643.523840] env[64020]: DEBUG oslo_concurrency.lockutils [req-3a17deca-181a-4620-b774-0b2fcf1caf92 req-0665c027-dff0-46b4-9659-1773e1cdcf2e service nova] Acquiring lock "refresh_cache-ae57ce0e-46b5-4490-af3f-4b2537bf316a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 643.689993] env[64020]: DEBUG nova.network.neutron [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 643.785971] env[64020]: DEBUG nova.network.neutron [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 643.786709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 32ac983fbff44af996ffd5a00bef33a9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 643.801036] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 32ac983fbff44af996ffd5a00bef33a9 [ 643.852796] env[64020]: DEBUG oslo_concurrency.lockutils [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 643.916488] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.688s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 643.916937] env[64020]: DEBUG nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 643.918577] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 617d02a085414cf5a19fdab6cdbc6295 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 643.919668] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.056s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 643.921196] env[64020]: INFO nova.compute.claims [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 643.923726] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg b38933d27e7b49188c3f84053d7191ce in queue reply_57893177120949e6a93cb88e15cd42b4 [ 643.942938] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 8909e440d10d41d7b9f37e71e6455a1d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 643.974594] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b38933d27e7b49188c3f84053d7191ce [ 643.983739] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 617d02a085414cf5a19fdab6cdbc6295 [ 644.005701] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8909e440d10d41d7b9f37e71e6455a1d [ 644.288445] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Releasing lock "refresh_cache-ae57ce0e-46b5-4490-af3f-4b2537bf316a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 644.289075] env[64020]: DEBUG nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 644.289232] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 644.289572] env[64020]: DEBUG oslo_concurrency.lockutils [req-3a17deca-181a-4620-b774-0b2fcf1caf92 req-0665c027-dff0-46b4-9659-1773e1cdcf2e service nova] Acquired lock "refresh_cache-ae57ce0e-46b5-4490-af3f-4b2537bf316a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 644.289741] env[64020]: DEBUG nova.network.neutron [req-3a17deca-181a-4620-b774-0b2fcf1caf92 req-0665c027-dff0-46b4-9659-1773e1cdcf2e service nova] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Refreshing network info cache for port 860ce71a-5c1b-48d2-8d62-3545e5714ce4 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 644.290177] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3a17deca-181a-4620-b774-0b2fcf1caf92 req-0665c027-dff0-46b4-9659-1773e1cdcf2e service nova] Expecting reply to msg fc63e030703a4a81ac9a1befd939cae6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 644.290991] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-21936ff8-8fca-48e0-aa77-f91eb1fa3eae {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 644.298647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc63e030703a4a81ac9a1befd939cae6 [ 644.301957] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94f5f495-8b07-4db7-be8b-ef9133ac96b0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 644.327726] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ae57ce0e-46b5-4490-af3f-4b2537bf316a could not be found. [ 644.327967] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 644.328238] env[64020]: INFO nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Took 0.04 seconds to destroy the instance on the hypervisor. [ 644.328402] env[64020]: DEBUG oslo.service.loopingcall [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 644.328623] env[64020]: DEBUG nova.compute.manager [-] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 644.328737] env[64020]: DEBUG nova.network.neutron [-] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 644.435114] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg 1e76f0114cbe44029cd8db44f8604fad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 644.436797] env[64020]: DEBUG nova.compute.utils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 644.437348] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 28377db32034439094991d6220242687 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 644.439373] env[64020]: DEBUG nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 644.439547] env[64020]: DEBUG nova.network.neutron [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 644.444548] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e76f0114cbe44029cd8db44f8604fad [ 644.445296] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Acquiring lock "b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 644.445522] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Lock "b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 644.448247] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28377db32034439094991d6220242687 [ 644.466380] env[64020]: INFO nova.scheduler.client.report [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Deleted allocations for instance 1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c [ 644.471958] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Expecting reply to msg 95077f1350db4562b8999d7832f63821 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 644.483409] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95077f1350db4562b8999d7832f63821 [ 644.561746] env[64020]: DEBUG nova.network.neutron [-] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 644.562315] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 20c34f99567d4eb4a312aac8852c0138 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 644.578813] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20c34f99567d4eb4a312aac8852c0138 [ 644.595673] env[64020]: DEBUG nova.policy [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2b175c17992b45f29fc1dd1bae5de8d4', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5dce267ed99746b39be0e2935c620dab', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 644.600751] env[64020]: DEBUG nova.compute.manager [req-2ed33be8-d4cf-4aa0-a276-f8675970205a req-f861f76e-2df6-4ac1-8753-a2bd2c27758d service nova] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Received event network-vif-deleted-1382c442-7ac3-48fc-b680-81c4ea9727f2 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 644.828702] env[64020]: DEBUG nova.network.neutron [req-3a17deca-181a-4620-b774-0b2fcf1caf92 req-0665c027-dff0-46b4-9659-1773e1cdcf2e service nova] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 644.912777] env[64020]: DEBUG nova.network.neutron [req-3a17deca-181a-4620-b774-0b2fcf1caf92 req-0665c027-dff0-46b4-9659-1773e1cdcf2e service nova] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 644.913318] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3a17deca-181a-4620-b774-0b2fcf1caf92 req-0665c027-dff0-46b4-9659-1773e1cdcf2e service nova] Expecting reply to msg 5fe3fb97853c4665a574232909df06c2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 644.922476] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5fe3fb97853c4665a574232909df06c2 [ 644.945635] env[64020]: DEBUG nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 644.947219] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg f0c52cc0f39041c089e7fd9af02ef92a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 644.976722] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b5b5ade-e094-4cce-a5ed-fc16cef98e1d tempest-ServerDiagnosticsNegativeTest-1587822607 tempest-ServerDiagnosticsNegativeTest-1587822607-project-member] Lock "1a31a6bc-0a7d-4a2b-bcd5-5680e828de6c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 51.412s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 644.976722] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 65ae62a28f4b4f54b5c70080800e2636 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 644.991580] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65ae62a28f4b4f54b5c70080800e2636 [ 644.999993] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f0c52cc0f39041c089e7fd9af02ef92a [ 645.065378] env[64020]: DEBUG nova.network.neutron [-] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 645.065499] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0e44c3f95660414d8f3c05ff0973d378 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 645.074613] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e44c3f95660414d8f3c05ff0973d378 [ 645.416666] env[64020]: DEBUG oslo_concurrency.lockutils [req-3a17deca-181a-4620-b774-0b2fcf1caf92 req-0665c027-dff0-46b4-9659-1773e1cdcf2e service nova] Releasing lock "refresh_cache-ae57ce0e-46b5-4490-af3f-4b2537bf316a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 645.452575] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 6274a0dd76fa49c69be145c85bfd27c0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 645.485323] env[64020]: DEBUG nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 645.487337] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 15eb5d5b56254762af3a6aa372804c43 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 645.489043] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5fbda1a8-d877-4427-81e1-e7169498730d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.497241] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d5c944d-eda1-4d0a-9e6e-a693404dffa3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.532748] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6274a0dd76fa49c69be145c85bfd27c0 [ 645.533337] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 15eb5d5b56254762af3a6aa372804c43 [ 645.534166] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3086dae-5ce2-4471-b39a-8b4c3a43ef0e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.543949] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c0d732d-7f5e-4e24-bb84-199924b5cf93 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.559805] env[64020]: DEBUG nova.compute.provider_tree [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 645.560375] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg c791f5364fdb4525849f3ab380c11f8f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 645.568337] env[64020]: INFO nova.compute.manager [-] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Took 1.24 seconds to deallocate network for instance. [ 645.568523] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c791f5364fdb4525849f3ab380c11f8f [ 645.570556] env[64020]: DEBUG nova.compute.claims [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 645.570724] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 645.749402] env[64020]: DEBUG nova.network.neutron [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Successfully created port: a872bec0-e59f-4427-a749-c89656be45fa {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 645.955967] env[64020]: DEBUG nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 645.985260] env[64020]: DEBUG nova.virt.hardware [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 645.985547] env[64020]: DEBUG nova.virt.hardware [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 645.985703] env[64020]: DEBUG nova.virt.hardware [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 645.985878] env[64020]: DEBUG nova.virt.hardware [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 645.986021] env[64020]: DEBUG nova.virt.hardware [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 645.986163] env[64020]: DEBUG nova.virt.hardware [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 645.986367] env[64020]: DEBUG nova.virt.hardware [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 645.986522] env[64020]: DEBUG nova.virt.hardware [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 645.986729] env[64020]: DEBUG nova.virt.hardware [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 645.986898] env[64020]: DEBUG nova.virt.hardware [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 645.987066] env[64020]: DEBUG nova.virt.hardware [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 645.987912] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87c17be2-f68e-4697-b197-ddcf28f82008 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.998756] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9d78a01-6f3b-44a1-b3b6-f2cc459d0073 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 646.034119] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.066650] env[64020]: DEBUG nova.scheduler.client.report [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 646.071320] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg ef64f830ddb0454a846aa3a3a9d70181 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 646.088320] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef64f830ddb0454a846aa3a3a9d70181 [ 646.573799] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.654s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.574331] env[64020]: DEBUG nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 646.576170] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg c72bee13a54d430db9c60bac4d34d4f4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 646.577192] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.716s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.578618] env[64020]: INFO nova.compute.claims [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 646.580455] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg a161e419de434da1b26880c9c845dd39 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 646.610199] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c72bee13a54d430db9c60bac4d34d4f4 [ 646.623713] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a161e419de434da1b26880c9c845dd39 [ 646.717569] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg f0a36962feaf4e6586f09096c3c5c13c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 646.728209] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f0a36962feaf4e6586f09096c3c5c13c [ 647.084025] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 3c7114bc1d1c4fada1a970c6d1bd941a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 647.085760] env[64020]: DEBUG nova.compute.utils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 647.086404] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg 5a98ec79945c41edb6cfd77f57d5a9d0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 647.087512] env[64020]: DEBUG nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 647.087512] env[64020]: DEBUG nova.network.neutron [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 647.093374] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c7114bc1d1c4fada1a970c6d1bd941a [ 647.096121] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a98ec79945c41edb6cfd77f57d5a9d0 [ 647.219537] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquiring lock "b3440b55-2469-46fa-ac2c-3e207bf530ec" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.219788] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "b3440b55-2469-46fa-ac2c-3e207bf530ec" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 647.224166] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquiring lock "b3440b55-2469-46fa-ac2c-3e207bf530ec-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.224166] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "b3440b55-2469-46fa-ac2c-3e207bf530ec-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 647.224166] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "b3440b55-2469-46fa-ac2c-3e207bf530ec-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 647.224166] env[64020]: DEBUG nova.compute.manager [req-75b3fcab-d0bc-469e-bc0d-361727487796 req-acbdb461-29b6-4af1-a689-5e78db32a27a service nova] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Received event network-vif-deleted-860ce71a-5c1b-48d2-8d62-3545e5714ce4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 647.224718] env[64020]: INFO nova.compute.manager [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Terminating instance [ 647.232020] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquiring lock "refresh_cache-b3440b55-2469-46fa-ac2c-3e207bf530ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 647.232020] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquired lock "refresh_cache-b3440b55-2469-46fa-ac2c-3e207bf530ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 647.232020] env[64020]: DEBUG nova.network.neutron [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 647.232020] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 98d8860c31544d279fd99d505d0c197b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 647.234292] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98d8860c31544d279fd99d505d0c197b [ 647.236373] env[64020]: DEBUG nova.policy [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a2b36482273349d4ac4c9fee2048ae13', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c5d12b1fa09846699c01ebd0f886f6e9', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 647.591214] env[64020]: DEBUG nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 647.593718] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg b3fd376cc6b24f1e8981961ffac8f00c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 647.632958] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3fd376cc6b24f1e8981961ffac8f00c [ 647.745188] env[64020]: DEBUG nova.network.neutron [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 647.796148] env[64020]: DEBUG nova.network.neutron [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 647.796655] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg e20b36fecff54dcb9a8fe3ab72863d01 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 647.808209] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e20b36fecff54dcb9a8fe3ab72863d01 [ 648.081349] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2af459e5-38a5-4bd4-b6bb-73ac6d0c6a57 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.092265] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18a05d83-9b18-459d-a205-ade4dcc04e10 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.099686] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg ffb08969e0b7435897cbd91d9cd801c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 648.138217] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4f2e1c9-fb8c-4ab1-ba3d-3f4b2b3eb774 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.141470] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ffb08969e0b7435897cbd91d9cd801c8 [ 648.147857] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8500c01b-f9bd-466c-aa4a-769611d64100 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.162123] env[64020]: DEBUG nova.compute.provider_tree [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 648.162667] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 2f45dbfef3644de68a71dbd5b376f1ff in queue reply_57893177120949e6a93cb88e15cd42b4 [ 648.170154] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f45dbfef3644de68a71dbd5b376f1ff [ 648.299714] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Releasing lock "refresh_cache-b3440b55-2469-46fa-ac2c-3e207bf530ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 648.300158] env[64020]: DEBUG nova.compute.manager [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 648.300348] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 648.301951] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-310aa527-a317-49f6-b16e-cd4961050179 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.309906] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 648.310162] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-8748eda8-7302-469c-b093-303b4d930de4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.317223] env[64020]: DEBUG oslo_vmware.api [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 648.317223] env[64020]: value = "task-407863" [ 648.317223] env[64020]: _type = "Task" [ 648.317223] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 648.329641] env[64020]: DEBUG oslo_vmware.api [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407863, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 648.368850] env[64020]: DEBUG nova.network.neutron [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Successfully created port: 59e1b8cf-9c47-43ad-8b36-bef6d9a79790 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 648.580910] env[64020]: ERROR nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a872bec0-e59f-4427-a749-c89656be45fa, please check neutron logs for more information. [ 648.580910] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 648.580910] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 648.580910] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 648.580910] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 648.580910] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 648.580910] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 648.580910] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 648.580910] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 648.580910] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 648.580910] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 648.580910] env[64020]: ERROR nova.compute.manager raise self.value [ 648.580910] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 648.580910] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 648.580910] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 648.580910] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 648.581527] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 648.581527] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 648.581527] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a872bec0-e59f-4427-a749-c89656be45fa, please check neutron logs for more information. [ 648.581527] env[64020]: ERROR nova.compute.manager [ 648.581527] env[64020]: Traceback (most recent call last): [ 648.581527] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 648.581527] env[64020]: listener.cb(fileno) [ 648.581527] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 648.581527] env[64020]: result = function(*args, **kwargs) [ 648.581527] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 648.581527] env[64020]: return func(*args, **kwargs) [ 648.581527] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 648.581527] env[64020]: raise e [ 648.581527] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 648.581527] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 648.581527] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 648.581527] env[64020]: created_port_ids = self._update_ports_for_instance( [ 648.581527] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 648.581527] env[64020]: with excutils.save_and_reraise_exception(): [ 648.581527] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 648.581527] env[64020]: self.force_reraise() [ 648.581527] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 648.581527] env[64020]: raise self.value [ 648.581527] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 648.581527] env[64020]: updated_port = self._update_port( [ 648.581527] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 648.581527] env[64020]: _ensure_no_port_binding_failure(port) [ 648.581527] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 648.581527] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 648.582584] env[64020]: nova.exception.PortBindingFailed: Binding failed for port a872bec0-e59f-4427-a749-c89656be45fa, please check neutron logs for more information. [ 648.582584] env[64020]: Removing descriptor: 16 [ 648.582584] env[64020]: ERROR nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a872bec0-e59f-4427-a749-c89656be45fa, please check neutron logs for more information. [ 648.582584] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Traceback (most recent call last): [ 648.582584] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 648.582584] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] yield resources [ 648.582584] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 648.582584] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] self.driver.spawn(context, instance, image_meta, [ 648.582584] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 648.582584] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] self._vmops.spawn(context, instance, image_meta, injected_files, [ 648.582584] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 648.582584] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] vm_ref = self.build_virtual_machine(instance, [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] vif_infos = vmwarevif.get_vif_info(self._session, [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] for vif in network_info: [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] return self._sync_wrapper(fn, *args, **kwargs) [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] self.wait() [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] self[:] = self._gt.wait() [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] return self._exit_event.wait() [ 648.582979] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] result = hub.switch() [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] return self.greenlet.switch() [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] result = function(*args, **kwargs) [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] return func(*args, **kwargs) [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] raise e [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] nwinfo = self.network_api.allocate_for_instance( [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 648.583412] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] created_port_ids = self._update_ports_for_instance( [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] with excutils.save_and_reraise_exception(): [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] self.force_reraise() [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] raise self.value [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] updated_port = self._update_port( [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] _ensure_no_port_binding_failure(port) [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 648.583830] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] raise exception.PortBindingFailed(port_id=port['id']) [ 648.584359] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] nova.exception.PortBindingFailed: Binding failed for port a872bec0-e59f-4427-a749-c89656be45fa, please check neutron logs for more information. [ 648.584359] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] [ 648.584359] env[64020]: INFO nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Terminating instance [ 648.585494] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquiring lock "refresh_cache-76de4647-9281-4c49-b4f1-16f0ea2ff313" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 648.585662] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquired lock "refresh_cache-76de4647-9281-4c49-b4f1-16f0ea2ff313" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 648.585825] env[64020]: DEBUG nova.network.neutron [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 648.586247] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg b746cf8bd9ad499cb2811e6f169fea07 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 648.594762] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b746cf8bd9ad499cb2811e6f169fea07 [ 648.602885] env[64020]: DEBUG nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 648.635789] env[64020]: DEBUG nova.virt.hardware [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 648.636067] env[64020]: DEBUG nova.virt.hardware [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 648.636225] env[64020]: DEBUG nova.virt.hardware [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 648.636436] env[64020]: DEBUG nova.virt.hardware [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 648.636820] env[64020]: DEBUG nova.virt.hardware [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 648.636982] env[64020]: DEBUG nova.virt.hardware [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 648.637191] env[64020]: DEBUG nova.virt.hardware [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 648.637349] env[64020]: DEBUG nova.virt.hardware [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 648.637509] env[64020]: DEBUG nova.virt.hardware [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 648.637664] env[64020]: DEBUG nova.virt.hardware [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 648.637839] env[64020]: DEBUG nova.virt.hardware [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 648.638710] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc03a6ef-4215-4d43-9be9-4fb8d683a590 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.647551] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7acf4909-a8f5-4a26-a163-634f6ccbfa27 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.665652] env[64020]: DEBUG nova.scheduler.client.report [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 648.668428] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 62b454a4169f4fe0bfb1e46a5b91389d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 648.684040] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62b454a4169f4fe0bfb1e46a5b91389d [ 648.828589] env[64020]: DEBUG oslo_vmware.api [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407863, 'name': PowerOffVM_Task, 'duration_secs': 0.133314} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 648.828892] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 648.829067] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 648.829281] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-9bd0176c-18fc-4649-982a-c9b55d45ceaa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.857860] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 648.858109] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Deleting contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 648.858291] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleting the datastore file [datastore1] b3440b55-2469-46fa-ac2c-3e207bf530ec {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 648.858636] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-ecb77c60-1c8a-4ca4-a4ba-b8a701be84eb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.867313] env[64020]: DEBUG oslo_vmware.api [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for the task: (returnval){ [ 648.867313] env[64020]: value = "task-407865" [ 648.867313] env[64020]: _type = "Task" [ 648.867313] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 648.878659] env[64020]: DEBUG oslo_vmware.api [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407865, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 649.115748] env[64020]: DEBUG nova.network.neutron [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 649.170884] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.594s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 649.171474] env[64020]: DEBUG nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 649.173404] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg dcc25297f0314b59a50bfc21e7db2187 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 649.174449] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.442s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 649.175878] env[64020]: INFO nova.compute.claims [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 649.177395] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg ad96430e99784147968f488cdc1da0c2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 649.203960] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dcc25297f0314b59a50bfc21e7db2187 [ 649.213980] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad96430e99784147968f488cdc1da0c2 [ 649.282583] env[64020]: DEBUG nova.network.neutron [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 649.283107] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 38603dcd4e494305bf17b0daab9b327c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 649.292098] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 38603dcd4e494305bf17b0daab9b327c [ 649.378957] env[64020]: DEBUG oslo_vmware.api [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Task: {'id': task-407865, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.093686} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 649.379257] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 649.379439] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Deleted contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 649.379609] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 649.379779] env[64020]: INFO nova.compute.manager [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Took 1.08 seconds to destroy the instance on the hypervisor. [ 649.380039] env[64020]: DEBUG oslo.service.loopingcall [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 649.380242] env[64020]: DEBUG nova.compute.manager [-] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 649.380400] env[64020]: DEBUG nova.network.neutron [-] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 649.416308] env[64020]: DEBUG nova.network.neutron [-] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 649.416833] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 063bef9cc1fb44a2a422904b89eb6488 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 649.425015] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 063bef9cc1fb44a2a422904b89eb6488 [ 649.680869] env[64020]: DEBUG nova.compute.utils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 649.681671] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 3e8b1a94dd2340bb84eb97be6bb4268d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 649.683042] env[64020]: DEBUG nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 649.683207] env[64020]: DEBUG nova.network.neutron [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 649.686938] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 12eedcb2a59a46569192098b480be9a5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 649.694097] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e8b1a94dd2340bb84eb97be6bb4268d [ 649.694696] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 12eedcb2a59a46569192098b480be9a5 [ 649.785359] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Releasing lock "refresh_cache-76de4647-9281-4c49-b4f1-16f0ea2ff313" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 649.785833] env[64020]: DEBUG nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 649.786047] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 649.786344] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d5e9b06b-3e7a-4613-9b0c-870d660fd929 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.797802] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ab4e2d1-c84e-4bea-96e4-9de8114d8ae3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.822383] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 76de4647-9281-4c49-b4f1-16f0ea2ff313 could not be found. [ 649.822604] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 649.822785] env[64020]: INFO nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Took 0.04 seconds to destroy the instance on the hypervisor. [ 649.823019] env[64020]: DEBUG oslo.service.loopingcall [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 649.823221] env[64020]: DEBUG nova.compute.manager [-] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 649.823311] env[64020]: DEBUG nova.network.neutron [-] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 649.839335] env[64020]: DEBUG nova.network.neutron [-] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 649.839802] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg d586a0cad6ec41f1b4cb81b758ff206a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 649.846682] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d586a0cad6ec41f1b4cb81b758ff206a [ 649.896942] env[64020]: DEBUG nova.policy [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '313f1212637d42edaf203fbd4b1955fe', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a4747df08b4844ba8f110ff7273d044e', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 649.919721] env[64020]: DEBUG nova.network.neutron [-] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 649.920211] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg e5d6c5a16b6a44838483c2939a74c91d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 649.928345] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e5d6c5a16b6a44838483c2939a74c91d [ 650.183846] env[64020]: DEBUG nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 650.185849] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 28fc297246924c46addbf622cd0cb04d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 650.225405] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28fc297246924c46addbf622cd0cb04d [ 650.345383] env[64020]: DEBUG nova.network.neutron [-] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 650.345383] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg de8fc0e5c5704b8eabbd260dd3a48a2e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 650.357090] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de8fc0e5c5704b8eabbd260dd3a48a2e [ 650.422412] env[64020]: INFO nova.compute.manager [-] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Took 1.04 seconds to deallocate network for instance. [ 650.427615] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 05600bd6e91a441ea9fa639174cb6664 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 650.463208] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05600bd6e91a441ea9fa639174cb6664 [ 650.690978] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 1f6e8105e17a45eeaec0878efe13491b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 650.709196] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3a78ae59-8b96-4093-ba8f-c135df315fd5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.728976] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-147e6828-34f7-4047-b1e2-960305ed035c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.766553] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f6e8105e17a45eeaec0878efe13491b [ 650.767029] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-829d81cc-858d-4a65-8b73-886b2b6217b5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.776303] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f44c2dd8-dc10-4cb8-b207-e6448013f14f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.797201] env[64020]: DEBUG nova.compute.provider_tree [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 650.797201] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg b71b57c4c6e247c885d41ce1b666f67e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 650.806539] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b71b57c4c6e247c885d41ce1b666f67e [ 650.847779] env[64020]: INFO nova.compute.manager [-] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Took 1.02 seconds to deallocate network for instance. [ 650.850447] env[64020]: DEBUG nova.compute.claims [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 650.850663] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.931050] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.984624] env[64020]: ERROR nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 59e1b8cf-9c47-43ad-8b36-bef6d9a79790, please check neutron logs for more information. [ 650.984624] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 650.984624] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 650.984624] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 650.984624] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 650.984624] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 650.984624] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 650.984624] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 650.984624] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 650.984624] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 650.984624] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 650.984624] env[64020]: ERROR nova.compute.manager raise self.value [ 650.984624] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 650.984624] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 650.984624] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 650.984624] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 650.985382] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 650.985382] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 650.985382] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 59e1b8cf-9c47-43ad-8b36-bef6d9a79790, please check neutron logs for more information. [ 650.985382] env[64020]: ERROR nova.compute.manager [ 650.985382] env[64020]: Traceback (most recent call last): [ 650.985382] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 650.985382] env[64020]: listener.cb(fileno) [ 650.985382] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 650.985382] env[64020]: result = function(*args, **kwargs) [ 650.985382] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 650.985382] env[64020]: return func(*args, **kwargs) [ 650.985382] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 650.985382] env[64020]: raise e [ 650.985382] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 650.985382] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 650.985382] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 650.985382] env[64020]: created_port_ids = self._update_ports_for_instance( [ 650.985382] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 650.985382] env[64020]: with excutils.save_and_reraise_exception(): [ 650.985382] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 650.985382] env[64020]: self.force_reraise() [ 650.985382] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 650.985382] env[64020]: raise self.value [ 650.985382] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 650.985382] env[64020]: updated_port = self._update_port( [ 650.985382] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 650.985382] env[64020]: _ensure_no_port_binding_failure(port) [ 650.985382] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 650.985382] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 650.986378] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 59e1b8cf-9c47-43ad-8b36-bef6d9a79790, please check neutron logs for more information. [ 650.986378] env[64020]: Removing descriptor: 17 [ 650.986378] env[64020]: ERROR nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 59e1b8cf-9c47-43ad-8b36-bef6d9a79790, please check neutron logs for more information. [ 650.986378] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Traceback (most recent call last): [ 650.986378] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 650.986378] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] yield resources [ 650.986378] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 650.986378] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] self.driver.spawn(context, instance, image_meta, [ 650.986378] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 650.986378] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 650.986378] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 650.986378] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] vm_ref = self.build_virtual_machine(instance, [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] vif_infos = vmwarevif.get_vif_info(self._session, [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] for vif in network_info: [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] return self._sync_wrapper(fn, *args, **kwargs) [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] self.wait() [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] self[:] = self._gt.wait() [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] return self._exit_event.wait() [ 650.986814] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] result = hub.switch() [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] return self.greenlet.switch() [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] result = function(*args, **kwargs) [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] return func(*args, **kwargs) [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] raise e [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] nwinfo = self.network_api.allocate_for_instance( [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 650.987298] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] created_port_ids = self._update_ports_for_instance( [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] with excutils.save_and_reraise_exception(): [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] self.force_reraise() [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] raise self.value [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] updated_port = self._update_port( [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] _ensure_no_port_binding_failure(port) [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 650.987822] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] raise exception.PortBindingFailed(port_id=port['id']) [ 650.988272] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] nova.exception.PortBindingFailed: Binding failed for port 59e1b8cf-9c47-43ad-8b36-bef6d9a79790, please check neutron logs for more information. [ 650.988272] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] [ 650.988272] env[64020]: INFO nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Terminating instance [ 650.988272] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Acquiring lock "refresh_cache-bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 650.988272] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Acquired lock "refresh_cache-bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 650.988272] env[64020]: DEBUG nova.network.neutron [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 650.988580] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg 416ff562c57744d1a9672454cd293426 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 650.990604] env[64020]: DEBUG nova.network.neutron [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Successfully created port: 2d332da1-175a-4a4b-8822-5e571fb24234 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 650.996592] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 416ff562c57744d1a9672454cd293426 [ 651.195052] env[64020]: DEBUG nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 651.221722] env[64020]: DEBUG nova.virt.hardware [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 651.221984] env[64020]: DEBUG nova.virt.hardware [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 651.222160] env[64020]: DEBUG nova.virt.hardware [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 651.222343] env[64020]: DEBUG nova.virt.hardware [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 651.222483] env[64020]: DEBUG nova.virt.hardware [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 651.222625] env[64020]: DEBUG nova.virt.hardware [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 651.222835] env[64020]: DEBUG nova.virt.hardware [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 651.222988] env[64020]: DEBUG nova.virt.hardware [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 651.223152] env[64020]: DEBUG nova.virt.hardware [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 651.223311] env[64020]: DEBUG nova.virt.hardware [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 651.223503] env[64020]: DEBUG nova.virt.hardware [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 651.224729] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-502d41dd-50e5-4bff-a04a-560b52a3b9c2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.234534] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9c076a7-69f7-448a-9f98-507391829a98 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.297269] env[64020]: DEBUG nova.scheduler.client.report [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 651.300609] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg efd6443d69c147a6b28771def4c37b61 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 651.316739] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg efd6443d69c147a6b28771def4c37b61 [ 651.341805] env[64020]: DEBUG nova.compute.manager [req-66ac5381-ab1d-44c0-ba29-3de5fb563795 req-6bec6669-ebf1-4a78-ae25-8e2e5bc12715 service nova] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Received event network-changed-a872bec0-e59f-4427-a749-c89656be45fa {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 651.341990] env[64020]: DEBUG nova.compute.manager [req-66ac5381-ab1d-44c0-ba29-3de5fb563795 req-6bec6669-ebf1-4a78-ae25-8e2e5bc12715 service nova] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Refreshing instance network info cache due to event network-changed-a872bec0-e59f-4427-a749-c89656be45fa. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 651.342361] env[64020]: DEBUG oslo_concurrency.lockutils [req-66ac5381-ab1d-44c0-ba29-3de5fb563795 req-6bec6669-ebf1-4a78-ae25-8e2e5bc12715 service nova] Acquiring lock "refresh_cache-76de4647-9281-4c49-b4f1-16f0ea2ff313" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 651.342584] env[64020]: DEBUG oslo_concurrency.lockutils [req-66ac5381-ab1d-44c0-ba29-3de5fb563795 req-6bec6669-ebf1-4a78-ae25-8e2e5bc12715 service nova] Acquired lock "refresh_cache-76de4647-9281-4c49-b4f1-16f0ea2ff313" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 651.342890] env[64020]: DEBUG nova.network.neutron [req-66ac5381-ab1d-44c0-ba29-3de5fb563795 req-6bec6669-ebf1-4a78-ae25-8e2e5bc12715 service nova] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Refreshing network info cache for port a872bec0-e59f-4427-a749-c89656be45fa {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 651.343241] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-66ac5381-ab1d-44c0-ba29-3de5fb563795 req-6bec6669-ebf1-4a78-ae25-8e2e5bc12715 service nova] Expecting reply to msg 4f07de8621024cef8a2d1d16db1bc73c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 651.350242] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f07de8621024cef8a2d1d16db1bc73c [ 651.661714] env[64020]: DEBUG nova.network.neutron [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 651.803508] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.629s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 651.804031] env[64020]: DEBUG nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 651.805871] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 38c3b3b000564b9292272f148cf7d96b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 651.807409] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 19.050s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 651.807732] env[64020]: DEBUG nova.objects.instance [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Trying to apply a migration context that does not seem to be set for this instance {{(pid=64020) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 651.809338] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 8d18da4421e54e748c2415264406fee2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 651.849876] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 38c3b3b000564b9292272f148cf7d96b [ 651.851647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d18da4421e54e748c2415264406fee2 [ 651.865423] env[64020]: DEBUG nova.network.neutron [req-66ac5381-ab1d-44c0-ba29-3de5fb563795 req-6bec6669-ebf1-4a78-ae25-8e2e5bc12715 service nova] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 651.881486] env[64020]: DEBUG nova.network.neutron [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 651.881974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg b4fee127c277417d8564da93c37ace99 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 651.890539] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b4fee127c277417d8564da93c37ace99 [ 651.994777] env[64020]: DEBUG nova.network.neutron [req-66ac5381-ab1d-44c0-ba29-3de5fb563795 req-6bec6669-ebf1-4a78-ae25-8e2e5bc12715 service nova] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 651.995289] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-66ac5381-ab1d-44c0-ba29-3de5fb563795 req-6bec6669-ebf1-4a78-ae25-8e2e5bc12715 service nova] Expecting reply to msg 04ca5ef537514938a67d9ee01a073134 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 652.005858] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 04ca5ef537514938a67d9ee01a073134 [ 652.308712] env[64020]: DEBUG nova.compute.utils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 652.309410] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg e8dd26e402b7488584de3351e1531d59 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 652.310420] env[64020]: DEBUG nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 652.310677] env[64020]: DEBUG nova.network.neutron [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 652.315527] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 9a24cc68756c4525acb4f72f4acb5257 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 652.323373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e8dd26e402b7488584de3351e1531d59 [ 652.324025] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9a24cc68756c4525acb4f72f4acb5257 [ 652.368123] env[64020]: DEBUG nova.policy [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ed9a4b5367c041679ece9c9b4d174297', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f36b9803b80d42bbaa4d6c1185335cd8', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 652.384214] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Releasing lock "refresh_cache-bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 652.384977] env[64020]: DEBUG nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 652.384977] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 652.385188] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-31f636c1-4c3d-4651-9b43-cda4fb090e16 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.395119] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6a236bd-2efa-4fe3-a5df-529c7175a818 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.424435] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance bfb84a86-8de0-4b2d-9355-85e0d8b5eba2 could not be found. [ 652.424715] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 652.424894] env[64020]: INFO nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Took 0.04 seconds to destroy the instance on the hypervisor. [ 652.425384] env[64020]: DEBUG oslo.service.loopingcall [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 652.425644] env[64020]: DEBUG nova.compute.manager [-] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 652.425733] env[64020]: DEBUG nova.network.neutron [-] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 652.455074] env[64020]: DEBUG nova.network.neutron [-] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 652.455711] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 75672093d76b42ab94ad47a8ef87d758 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 652.463605] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 75672093d76b42ab94ad47a8ef87d758 [ 652.502708] env[64020]: DEBUG oslo_concurrency.lockutils [req-66ac5381-ab1d-44c0-ba29-3de5fb563795 req-6bec6669-ebf1-4a78-ae25-8e2e5bc12715 service nova] Releasing lock "refresh_cache-76de4647-9281-4c49-b4f1-16f0ea2ff313" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 652.503139] env[64020]: DEBUG nova.compute.manager [req-66ac5381-ab1d-44c0-ba29-3de5fb563795 req-6bec6669-ebf1-4a78-ae25-8e2e5bc12715 service nova] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Received event network-vif-deleted-a872bec0-e59f-4427-a749-c89656be45fa {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 652.721204] env[64020]: DEBUG nova.network.neutron [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Successfully created port: a1866c0a-948c-477e-b20f-1d7959b22ab6 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 652.816140] env[64020]: DEBUG nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 652.817873] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg c876bd7bd0c94a4bb19c36e40d25ff56 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 652.824398] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.017s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 652.824730] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ee0dc8d2-e0cd-41fa-b1e2-44816bd66071 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg c61b2e013c834484ac7617b629c1d395 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 652.825551] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.172s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 652.827004] env[64020]: INFO nova.compute.claims [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 652.836488] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg e9f8ad4bbdd44db5879170d97394b456 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 652.842638] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c61b2e013c834484ac7617b629c1d395 [ 652.877986] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c876bd7bd0c94a4bb19c36e40d25ff56 [ 652.892543] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9f8ad4bbdd44db5879170d97394b456 [ 652.957966] env[64020]: DEBUG nova.network.neutron [-] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 652.958464] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a6e3d94d6d7c473383b6123d5561cc6f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 652.967009] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6e3d94d6d7c473383b6123d5561cc6f [ 653.326025] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 49f0d860a57647f69b5224aa3f6b9b1b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 653.345698] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 016e93113f0c43d1bb7f8dca1d0ce3b4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 653.360199] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 016e93113f0c43d1bb7f8dca1d0ce3b4 [ 653.365698] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 49f0d860a57647f69b5224aa3f6b9b1b [ 653.454115] env[64020]: ERROR nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 2d332da1-175a-4a4b-8822-5e571fb24234, please check neutron logs for more information. [ 653.454115] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 653.454115] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 653.454115] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 653.454115] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 653.454115] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 653.454115] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 653.454115] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 653.454115] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 653.454115] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 653.454115] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 653.454115] env[64020]: ERROR nova.compute.manager raise self.value [ 653.454115] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 653.454115] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 653.454115] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 653.454115] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 653.454671] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 653.454671] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 653.454671] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 2d332da1-175a-4a4b-8822-5e571fb24234, please check neutron logs for more information. [ 653.454671] env[64020]: ERROR nova.compute.manager [ 653.454671] env[64020]: Traceback (most recent call last): [ 653.454671] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 653.454671] env[64020]: listener.cb(fileno) [ 653.454671] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 653.454671] env[64020]: result = function(*args, **kwargs) [ 653.454671] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 653.454671] env[64020]: return func(*args, **kwargs) [ 653.454671] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 653.454671] env[64020]: raise e [ 653.454671] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 653.454671] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 653.454671] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 653.454671] env[64020]: created_port_ids = self._update_ports_for_instance( [ 653.454671] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 653.454671] env[64020]: with excutils.save_and_reraise_exception(): [ 653.454671] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 653.454671] env[64020]: self.force_reraise() [ 653.454671] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 653.454671] env[64020]: raise self.value [ 653.454671] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 653.454671] env[64020]: updated_port = self._update_port( [ 653.454671] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 653.454671] env[64020]: _ensure_no_port_binding_failure(port) [ 653.454671] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 653.454671] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 653.455555] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 2d332da1-175a-4a4b-8822-5e571fb24234, please check neutron logs for more information. [ 653.455555] env[64020]: Removing descriptor: 16 [ 653.455555] env[64020]: ERROR nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 2d332da1-175a-4a4b-8822-5e571fb24234, please check neutron logs for more information. [ 653.455555] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Traceback (most recent call last): [ 653.455555] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 653.455555] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] yield resources [ 653.455555] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 653.455555] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] self.driver.spawn(context, instance, image_meta, [ 653.455555] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 653.455555] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] self._vmops.spawn(context, instance, image_meta, injected_files, [ 653.455555] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 653.455555] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] vm_ref = self.build_virtual_machine(instance, [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] vif_infos = vmwarevif.get_vif_info(self._session, [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] for vif in network_info: [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] return self._sync_wrapper(fn, *args, **kwargs) [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] self.wait() [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] self[:] = self._gt.wait() [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] return self._exit_event.wait() [ 653.456025] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] result = hub.switch() [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] return self.greenlet.switch() [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] result = function(*args, **kwargs) [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] return func(*args, **kwargs) [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] raise e [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] nwinfo = self.network_api.allocate_for_instance( [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 653.456430] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] created_port_ids = self._update_ports_for_instance( [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] with excutils.save_and_reraise_exception(): [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] self.force_reraise() [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] raise self.value [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] updated_port = self._update_port( [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] _ensure_no_port_binding_failure(port) [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 653.456846] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] raise exception.PortBindingFailed(port_id=port['id']) [ 653.457247] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] nova.exception.PortBindingFailed: Binding failed for port 2d332da1-175a-4a4b-8822-5e571fb24234, please check neutron logs for more information. [ 653.457247] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] [ 653.457247] env[64020]: INFO nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Terminating instance [ 653.457840] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Acquiring lock "refresh_cache-618c3b06-1fd8-45d3-9c59-61c0e202a299" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 653.457998] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Acquired lock "refresh_cache-618c3b06-1fd8-45d3-9c59-61c0e202a299" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 653.458155] env[64020]: DEBUG nova.network.neutron [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 653.458705] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 0b9ed27bd7c943a0acedc92bdb510081 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 653.461776] env[64020]: INFO nova.compute.manager [-] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Took 1.04 seconds to deallocate network for instance. [ 653.462887] env[64020]: DEBUG nova.compute.claims [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 653.463079] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 653.466229] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b9ed27bd7c943a0acedc92bdb510081 [ 653.834509] env[64020]: DEBUG nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 653.864730] env[64020]: DEBUG nova.virt.hardware [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 653.865610] env[64020]: DEBUG nova.virt.hardware [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 653.865984] env[64020]: DEBUG nova.virt.hardware [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 653.866064] env[64020]: DEBUG nova.virt.hardware [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 653.866216] env[64020]: DEBUG nova.virt.hardware [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 653.866474] env[64020]: DEBUG nova.virt.hardware [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 653.866794] env[64020]: DEBUG nova.virt.hardware [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 653.867034] env[64020]: DEBUG nova.virt.hardware [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 653.867222] env[64020]: DEBUG nova.virt.hardware [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 653.867417] env[64020]: DEBUG nova.virt.hardware [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 653.867616] env[64020]: DEBUG nova.virt.hardware [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 653.868554] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1b642fa-4166-4d7d-bff0-16b381182dd1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.877704] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58e7a7a6-ac23-43fc-a5db-4199506aef1f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.993002] env[64020]: DEBUG nova.network.neutron [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 654.136082] env[64020]: DEBUG nova.network.neutron [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 654.136652] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg f811742ff8cc40f0be97687ada7d5587 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 654.145811] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f811742ff8cc40f0be97687ada7d5587 [ 654.357982] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f42a6a2-b45e-4a3d-b322-965ec3c011cd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.366186] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f814347-deb4-4a3e-9b47-f974e9cf5b2a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.398548] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdd14755-3c5a-4802-b2be-33331e28c7a8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.406365] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3f1c214-8539-4e99-9895-6a1a013579aa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.414124] env[64020]: DEBUG nova.compute.manager [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Received event network-changed-59e1b8cf-9c47-43ad-8b36-bef6d9a79790 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 654.414335] env[64020]: DEBUG nova.compute.manager [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Refreshing instance network info cache due to event network-changed-59e1b8cf-9c47-43ad-8b36-bef6d9a79790. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 654.414542] env[64020]: DEBUG oslo_concurrency.lockutils [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] Acquiring lock "refresh_cache-bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 654.414732] env[64020]: DEBUG oslo_concurrency.lockutils [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] Acquired lock "refresh_cache-bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 654.414938] env[64020]: DEBUG nova.network.neutron [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Refreshing network info cache for port 59e1b8cf-9c47-43ad-8b36-bef6d9a79790 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 654.422014] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] Expecting reply to msg 57d83514b5474ad5969360f4118c0c1b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 654.434849] env[64020]: DEBUG nova.compute.provider_tree [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 654.435338] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg ecf64fed38cd4cb0b0a6cb81f70a14f7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 654.436343] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 57d83514b5474ad5969360f4118c0c1b [ 654.443750] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ecf64fed38cd4cb0b0a6cb81f70a14f7 [ 654.639228] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Releasing lock "refresh_cache-618c3b06-1fd8-45d3-9c59-61c0e202a299" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 654.639228] env[64020]: DEBUG nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 654.639376] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 654.639820] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2fe64c43-fadf-4be2-ae46-6df6580fbba0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.649913] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-509d80c2-7647-44c0-ab22-3b49383f9f48 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.676027] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 618c3b06-1fd8-45d3-9c59-61c0e202a299 could not be found. [ 654.676027] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 654.676027] env[64020]: INFO nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Took 0.04 seconds to destroy the instance on the hypervisor. [ 654.676027] env[64020]: DEBUG oslo.service.loopingcall [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 654.679097] env[64020]: ERROR nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a1866c0a-948c-477e-b20f-1d7959b22ab6, please check neutron logs for more information. [ 654.679097] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 654.679097] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 654.679097] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 654.679097] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 654.679097] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 654.679097] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 654.679097] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 654.679097] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 654.679097] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 654.679097] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 654.679097] env[64020]: ERROR nova.compute.manager raise self.value [ 654.679097] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 654.679097] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 654.679097] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 654.679097] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 654.679612] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 654.679612] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 654.679612] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a1866c0a-948c-477e-b20f-1d7959b22ab6, please check neutron logs for more information. [ 654.679612] env[64020]: ERROR nova.compute.manager [ 654.679612] env[64020]: Traceback (most recent call last): [ 654.679612] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 654.679612] env[64020]: listener.cb(fileno) [ 654.679612] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 654.679612] env[64020]: result = function(*args, **kwargs) [ 654.679612] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 654.679612] env[64020]: return func(*args, **kwargs) [ 654.679612] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 654.679612] env[64020]: raise e [ 654.679612] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 654.679612] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 654.679612] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 654.679612] env[64020]: created_port_ids = self._update_ports_for_instance( [ 654.679612] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 654.679612] env[64020]: with excutils.save_and_reraise_exception(): [ 654.679612] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 654.679612] env[64020]: self.force_reraise() [ 654.679612] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 654.679612] env[64020]: raise self.value [ 654.679612] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 654.679612] env[64020]: updated_port = self._update_port( [ 654.679612] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 654.679612] env[64020]: _ensure_no_port_binding_failure(port) [ 654.679612] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 654.679612] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 654.680583] env[64020]: nova.exception.PortBindingFailed: Binding failed for port a1866c0a-948c-477e-b20f-1d7959b22ab6, please check neutron logs for more information. [ 654.680583] env[64020]: Removing descriptor: 18 [ 654.680583] env[64020]: DEBUG nova.compute.manager [-] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 654.680583] env[64020]: DEBUG nova.network.neutron [-] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 654.680583] env[64020]: ERROR nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a1866c0a-948c-477e-b20f-1d7959b22ab6, please check neutron logs for more information. [ 654.680583] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Traceback (most recent call last): [ 654.680583] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 654.680583] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] yield resources [ 654.680583] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 654.680583] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] self.driver.spawn(context, instance, image_meta, [ 654.680583] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] self._vmops.spawn(context, instance, image_meta, injected_files, [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] vm_ref = self.build_virtual_machine(instance, [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] vif_infos = vmwarevif.get_vif_info(self._session, [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] for vif in network_info: [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] return self._sync_wrapper(fn, *args, **kwargs) [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] self.wait() [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 654.681112] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] self[:] = self._gt.wait() [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] return self._exit_event.wait() [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] result = hub.switch() [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] return self.greenlet.switch() [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] result = function(*args, **kwargs) [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] return func(*args, **kwargs) [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] raise e [ 654.681580] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] nwinfo = self.network_api.allocate_for_instance( [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] created_port_ids = self._update_ports_for_instance( [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] with excutils.save_and_reraise_exception(): [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] self.force_reraise() [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] raise self.value [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] updated_port = self._update_port( [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 654.682406] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] _ensure_no_port_binding_failure(port) [ 654.682849] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 654.682849] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] raise exception.PortBindingFailed(port_id=port['id']) [ 654.682849] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] nova.exception.PortBindingFailed: Binding failed for port a1866c0a-948c-477e-b20f-1d7959b22ab6, please check neutron logs for more information. [ 654.682849] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] [ 654.682849] env[64020]: INFO nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Terminating instance [ 654.683971] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Acquiring lock "refresh_cache-4478746a-11a6-480d-84c1-c6f04ab33505" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 654.684184] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Acquired lock "refresh_cache-4478746a-11a6-480d-84c1-c6f04ab33505" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 654.684362] env[64020]: DEBUG nova.network.neutron [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 654.685223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg adc5698e78f9447995a98e0a7a906630 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 654.691457] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg adc5698e78f9447995a98e0a7a906630 [ 654.698455] env[64020]: DEBUG nova.network.neutron [-] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 654.698978] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg f3a538fa4b6a4e438c3d313ca20a31d2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 654.705970] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3a538fa4b6a4e438c3d313ca20a31d2 [ 654.856790] env[64020]: DEBUG nova.compute.manager [req-e0b10af3-46e5-4f04-95bd-3afba5b6d28c req-701e32a2-717d-48c2-885b-786d90fa66a1 service nova] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Received event network-changed-a1866c0a-948c-477e-b20f-1d7959b22ab6 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 654.857119] env[64020]: DEBUG nova.compute.manager [req-e0b10af3-46e5-4f04-95bd-3afba5b6d28c req-701e32a2-717d-48c2-885b-786d90fa66a1 service nova] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Refreshing instance network info cache due to event network-changed-a1866c0a-948c-477e-b20f-1d7959b22ab6. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 654.857441] env[64020]: DEBUG oslo_concurrency.lockutils [req-e0b10af3-46e5-4f04-95bd-3afba5b6d28c req-701e32a2-717d-48c2-885b-786d90fa66a1 service nova] Acquiring lock "refresh_cache-4478746a-11a6-480d-84c1-c6f04ab33505" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 654.938685] env[64020]: DEBUG nova.scheduler.client.report [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 654.941544] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg d150e8a32403421fa090d8bc8da7c0e6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 654.955312] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d150e8a32403421fa090d8bc8da7c0e6 [ 654.965001] env[64020]: DEBUG nova.network.neutron [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 655.091958] env[64020]: DEBUG nova.network.neutron [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.092605] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] Expecting reply to msg 67f30a39e1934e0b9b13112af9dbab4e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 655.100961] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 67f30a39e1934e0b9b13112af9dbab4e [ 655.200989] env[64020]: DEBUG nova.network.neutron [-] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.201518] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 83de56e9815f4bc0bd3d4ae9b9da7c43 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 655.205715] env[64020]: DEBUG nova.network.neutron [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 655.211821] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 83de56e9815f4bc0bd3d4ae9b9da7c43 [ 655.283728] env[64020]: DEBUG nova.network.neutron [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.284282] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 9b420d5a82014f99b5add284882c21be in queue reply_57893177120949e6a93cb88e15cd42b4 [ 655.293633] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b420d5a82014f99b5add284882c21be [ 655.444102] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.618s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 655.444670] env[64020]: DEBUG nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 655.446697] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg d7d832453cf64792b32a1527c4e559c9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 655.447915] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.778s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 655.449923] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg fb7875d78fd545d39d73dff3541bc171 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 655.486036] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7d832453cf64792b32a1527c4e559c9 [ 655.486730] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb7875d78fd545d39d73dff3541bc171 [ 655.595062] env[64020]: DEBUG oslo_concurrency.lockutils [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] Releasing lock "refresh_cache-bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 655.595343] env[64020]: DEBUG nova.compute.manager [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Received event network-vif-deleted-59e1b8cf-9c47-43ad-8b36-bef6d9a79790 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 655.595534] env[64020]: DEBUG nova.compute.manager [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Received event network-changed-2d332da1-175a-4a4b-8822-5e571fb24234 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 655.595690] env[64020]: DEBUG nova.compute.manager [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Refreshing instance network info cache due to event network-changed-2d332da1-175a-4a4b-8822-5e571fb24234. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 655.595939] env[64020]: DEBUG oslo_concurrency.lockutils [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] Acquiring lock "refresh_cache-618c3b06-1fd8-45d3-9c59-61c0e202a299" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 655.596031] env[64020]: DEBUG oslo_concurrency.lockutils [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] Acquired lock "refresh_cache-618c3b06-1fd8-45d3-9c59-61c0e202a299" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 655.596189] env[64020]: DEBUG nova.network.neutron [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Refreshing network info cache for port 2d332da1-175a-4a4b-8822-5e571fb24234 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 655.596619] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] Expecting reply to msg 84e25b3d5dbf4018801985e84f405122 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 655.607465] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 84e25b3d5dbf4018801985e84f405122 [ 655.703914] env[64020]: INFO nova.compute.manager [-] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Took 1.03 seconds to deallocate network for instance. [ 655.706241] env[64020]: DEBUG nova.compute.claims [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 655.706451] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 655.787123] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Releasing lock "refresh_cache-4478746a-11a6-480d-84c1-c6f04ab33505" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 655.787818] env[64020]: DEBUG nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 655.788196] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 655.788655] env[64020]: DEBUG oslo_concurrency.lockutils [req-e0b10af3-46e5-4f04-95bd-3afba5b6d28c req-701e32a2-717d-48c2-885b-786d90fa66a1 service nova] Acquired lock "refresh_cache-4478746a-11a6-480d-84c1-c6f04ab33505" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 655.788983] env[64020]: DEBUG nova.network.neutron [req-e0b10af3-46e5-4f04-95bd-3afba5b6d28c req-701e32a2-717d-48c2-885b-786d90fa66a1 service nova] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Refreshing network info cache for port a1866c0a-948c-477e-b20f-1d7959b22ab6 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 655.789683] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e0b10af3-46e5-4f04-95bd-3afba5b6d28c req-701e32a2-717d-48c2-885b-786d90fa66a1 service nova] Expecting reply to msg 6c4d7193212740bab5e2b238822a78c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 655.791120] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-66d010ec-833b-463b-a9f1-8a4b28582806 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.806164] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6c4d7193212740bab5e2b238822a78c8 [ 655.818730] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd1791b1-2458-4e53-9e2a-d17adbe0b7c5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.829387] env[64020]: DEBUG nova.network.neutron [req-e0b10af3-46e5-4f04-95bd-3afba5b6d28c req-701e32a2-717d-48c2-885b-786d90fa66a1 service nova] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 655.851632] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4478746a-11a6-480d-84c1-c6f04ab33505 could not be found. [ 655.852047] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 655.852161] env[64020]: INFO nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Took 0.06 seconds to destroy the instance on the hypervisor. [ 655.852459] env[64020]: DEBUG oslo.service.loopingcall [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 655.852800] env[64020]: DEBUG nova.compute.manager [-] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 655.852948] env[64020]: DEBUG nova.network.neutron [-] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 655.882274] env[64020]: DEBUG nova.network.neutron [-] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 655.882810] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 198d9b709a7d4fe6a2736e32a6191b5a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 655.890405] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 198d9b709a7d4fe6a2736e32a6191b5a [ 655.908617] env[64020]: DEBUG nova.network.neutron [req-e0b10af3-46e5-4f04-95bd-3afba5b6d28c req-701e32a2-717d-48c2-885b-786d90fa66a1 service nova] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.909129] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e0b10af3-46e5-4f04-95bd-3afba5b6d28c req-701e32a2-717d-48c2-885b-786d90fa66a1 service nova] Expecting reply to msg 22e3430842604383ab204fb52f2730b4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 655.916518] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 22e3430842604383ab204fb52f2730b4 [ 655.953361] env[64020]: DEBUG nova.compute.utils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 655.954075] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 866c21543f474bcab11b2513cfc2ae84 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 655.958203] env[64020]: DEBUG nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 655.958561] env[64020]: DEBUG nova.network.neutron [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 655.966394] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 866c21543f474bcab11b2513cfc2ae84 [ 655.998660] env[64020]: DEBUG nova.policy [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ab279e2eace8446eb3597a8c117c2e6b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '030c8900b5ef4767846f0740cbbb15be', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 656.126434] env[64020]: DEBUG nova.network.neutron [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 656.271293] env[64020]: DEBUG nova.network.neutron [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 656.271646] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] Expecting reply to msg 21d38254aa854464a61f4609af31a8dd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 656.280631] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21d38254aa854464a61f4609af31a8dd [ 656.375513] env[64020]: DEBUG nova.network.neutron [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Successfully created port: 89bd3d47-7a39-459a-bf81-51dab50c2f84 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 656.385136] env[64020]: DEBUG nova.network.neutron [-] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 656.385665] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1d462b5c06174642b30dc26304363cb8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 656.393933] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d462b5c06174642b30dc26304363cb8 [ 656.410909] env[64020]: DEBUG oslo_concurrency.lockutils [req-e0b10af3-46e5-4f04-95bd-3afba5b6d28c req-701e32a2-717d-48c2-885b-786d90fa66a1 service nova] Releasing lock "refresh_cache-4478746a-11a6-480d-84c1-c6f04ab33505" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 656.458545] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3895d4c0-e3af-4b89-a32b-d0328feab53f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 656.461827] env[64020]: DEBUG nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 656.463821] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 0117c37e6c9349a0ba4756ad2f3bb95e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 656.470748] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd86166c-a255-4172-ac03-dacb17321b4a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 656.503771] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0117c37e6c9349a0ba4756ad2f3bb95e [ 656.504696] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2401e935-cddf-4080-bf24-d39b6d558a78 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 656.514221] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d573d6b3-d176-4a9d-ab3d-fd5265114949 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 656.528684] env[64020]: DEBUG nova.compute.provider_tree [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 656.529243] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 1d4ef523df1f481698263954a97b9eb6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 656.536525] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d4ef523df1f481698263954a97b9eb6 [ 656.662316] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Acquiring lock "1954e745-783e-417e-aafc-265da55066cf" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 656.662668] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Lock "1954e745-783e-417e-aafc-265da55066cf" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 656.774588] env[64020]: DEBUG oslo_concurrency.lockutils [req-de87cbba-0b13-4922-a984-a1da6bd464b9 req-9317024d-a65d-4b8f-8718-f723730f981b service nova] Releasing lock "refresh_cache-618c3b06-1fd8-45d3-9c59-61c0e202a299" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 656.873175] env[64020]: DEBUG nova.compute.manager [req-53c94c26-3a35-4894-b90b-9a18183d2e60 req-4edd9b58-e047-4216-9528-50442031d14a service nova] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Received event network-vif-deleted-2d332da1-175a-4a4b-8822-5e571fb24234 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 656.887716] env[64020]: INFO nova.compute.manager [-] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Took 1.03 seconds to deallocate network for instance. [ 656.890128] env[64020]: DEBUG nova.compute.claims [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 656.890307] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 656.968629] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg d22f2783d9fa471ba53a9c49e1a80c0c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 657.006063] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d22f2783d9fa471ba53a9c49e1a80c0c [ 657.031387] env[64020]: DEBUG nova.scheduler.client.report [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 657.033809] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 088bbe4528af4298b53e8ca359066653 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 657.048358] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 088bbe4528af4298b53e8ca359066653 [ 657.148493] env[64020]: DEBUG nova.compute.manager [req-088608b0-37cb-4dad-b897-a0e171b32d6a req-58e385e9-b56c-4125-bc80-d4ad4fd4efc7 service nova] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Received event network-vif-deleted-a1866c0a-948c-477e-b20f-1d7959b22ab6 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 657.303125] env[64020]: ERROR nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 89bd3d47-7a39-459a-bf81-51dab50c2f84, please check neutron logs for more information. [ 657.303125] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 657.303125] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 657.303125] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 657.303125] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 657.303125] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 657.303125] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 657.303125] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 657.303125] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 657.303125] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 657.303125] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 657.303125] env[64020]: ERROR nova.compute.manager raise self.value [ 657.303125] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 657.303125] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 657.303125] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 657.303125] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 657.303983] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 657.303983] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 657.303983] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 89bd3d47-7a39-459a-bf81-51dab50c2f84, please check neutron logs for more information. [ 657.303983] env[64020]: ERROR nova.compute.manager [ 657.303983] env[64020]: Traceback (most recent call last): [ 657.303983] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 657.303983] env[64020]: listener.cb(fileno) [ 657.303983] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 657.303983] env[64020]: result = function(*args, **kwargs) [ 657.303983] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 657.303983] env[64020]: return func(*args, **kwargs) [ 657.303983] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 657.303983] env[64020]: raise e [ 657.303983] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 657.303983] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 657.303983] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 657.303983] env[64020]: created_port_ids = self._update_ports_for_instance( [ 657.303983] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 657.303983] env[64020]: with excutils.save_and_reraise_exception(): [ 657.303983] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 657.303983] env[64020]: self.force_reraise() [ 657.303983] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 657.303983] env[64020]: raise self.value [ 657.303983] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 657.303983] env[64020]: updated_port = self._update_port( [ 657.303983] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 657.303983] env[64020]: _ensure_no_port_binding_failure(port) [ 657.303983] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 657.303983] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 657.305003] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 89bd3d47-7a39-459a-bf81-51dab50c2f84, please check neutron logs for more information. [ 657.305003] env[64020]: Removing descriptor: 18 [ 657.474753] env[64020]: DEBUG nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 657.527931] env[64020]: DEBUG nova.virt.hardware [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 657.528383] env[64020]: DEBUG nova.virt.hardware [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 657.528633] env[64020]: DEBUG nova.virt.hardware [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 657.528934] env[64020]: DEBUG nova.virt.hardware [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 657.529175] env[64020]: DEBUG nova.virt.hardware [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 657.529426] env[64020]: DEBUG nova.virt.hardware [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 657.529766] env[64020]: DEBUG nova.virt.hardware [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 657.530027] env[64020]: DEBUG nova.virt.hardware [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 657.530281] env[64020]: DEBUG nova.virt.hardware [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 657.530551] env[64020]: DEBUG nova.virt.hardware [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 657.531244] env[64020]: DEBUG nova.virt.hardware [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 657.532492] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-729f9adc-8cc9-485c-9835-aeee3ecff222 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 657.536613] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.089s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 657.538694] env[64020]: ERROR nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b3f5c844-2c18-4084-9948-08386b4b5276, please check neutron logs for more information. [ 657.538694] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Traceback (most recent call last): [ 657.538694] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 657.538694] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] self.driver.spawn(context, instance, image_meta, [ 657.538694] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 657.538694] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] self._vmops.spawn(context, instance, image_meta, injected_files, [ 657.538694] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 657.538694] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] vm_ref = self.build_virtual_machine(instance, [ 657.538694] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 657.538694] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] vif_infos = vmwarevif.get_vif_info(self._session, [ 657.538694] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] for vif in network_info: [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] return self._sync_wrapper(fn, *args, **kwargs) [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] self.wait() [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] self[:] = self._gt.wait() [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] return self._exit_event.wait() [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] result = hub.switch() [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 657.539302] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] return self.greenlet.switch() [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] result = function(*args, **kwargs) [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] return func(*args, **kwargs) [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] raise e [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] nwinfo = self.network_api.allocate_for_instance( [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] created_port_ids = self._update_ports_for_instance( [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] with excutils.save_and_reraise_exception(): [ 657.539889] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 657.540425] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] self.force_reraise() [ 657.540425] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 657.540425] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] raise self.value [ 657.540425] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 657.540425] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] updated_port = self._update_port( [ 657.540425] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 657.540425] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] _ensure_no_port_binding_failure(port) [ 657.540425] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 657.540425] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] raise exception.PortBindingFailed(port_id=port['id']) [ 657.540425] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] nova.exception.PortBindingFailed: Binding failed for port b3f5c844-2c18-4084-9948-08386b4b5276, please check neutron logs for more information. [ 657.540425] env[64020]: ERROR nova.compute.manager [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] [ 657.543428] env[64020]: DEBUG nova.compute.utils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Binding failed for port b3f5c844-2c18-4084-9948-08386b4b5276, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 657.545517] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.511s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 657.547226] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg b6cc4b2b9995461080a29a60a45c6645 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 657.549244] env[64020]: DEBUG nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Build of instance 65be7d88-0c12-4ad3-adc7-d089a1b91c28 was re-scheduled: Binding failed for port b3f5c844-2c18-4084-9948-08386b4b5276, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 657.550016] env[64020]: DEBUG nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 657.550346] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquiring lock "refresh_cache-65be7d88-0c12-4ad3-adc7-d089a1b91c28" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 657.550676] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquired lock "refresh_cache-65be7d88-0c12-4ad3-adc7-d089a1b91c28" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 657.551030] env[64020]: DEBUG nova.network.neutron [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 657.551686] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 60e8d9f4374a4c6abfbb9a197d907dd3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 657.557137] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4b04e3d-6117-4962-a734-4ad3f4094ede {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 657.569266] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60e8d9f4374a4c6abfbb9a197d907dd3 [ 657.581822] env[64020]: ERROR nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 89bd3d47-7a39-459a-bf81-51dab50c2f84, please check neutron logs for more information. [ 657.581822] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Traceback (most recent call last): [ 657.581822] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 657.581822] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] yield resources [ 657.581822] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 657.581822] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] self.driver.spawn(context, instance, image_meta, [ 657.581822] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 657.581822] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 657.581822] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 657.581822] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] vm_ref = self.build_virtual_machine(instance, [ 657.581822] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] vif_infos = vmwarevif.get_vif_info(self._session, [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] for vif in network_info: [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] return self._sync_wrapper(fn, *args, **kwargs) [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] self.wait() [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] self[:] = self._gt.wait() [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] return self._exit_event.wait() [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 657.582411] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] current.throw(*self._exc) [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] result = function(*args, **kwargs) [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] return func(*args, **kwargs) [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] raise e [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] nwinfo = self.network_api.allocate_for_instance( [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] created_port_ids = self._update_ports_for_instance( [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] with excutils.save_and_reraise_exception(): [ 657.582882] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 657.583349] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] self.force_reraise() [ 657.583349] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 657.583349] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] raise self.value [ 657.583349] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 657.583349] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] updated_port = self._update_port( [ 657.583349] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 657.583349] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] _ensure_no_port_binding_failure(port) [ 657.583349] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 657.583349] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] raise exception.PortBindingFailed(port_id=port['id']) [ 657.583349] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] nova.exception.PortBindingFailed: Binding failed for port 89bd3d47-7a39-459a-bf81-51dab50c2f84, please check neutron logs for more information. [ 657.583349] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] [ 657.583349] env[64020]: INFO nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Terminating instance [ 657.586499] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Acquiring lock "refresh_cache-9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 657.586717] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Acquired lock "refresh_cache-9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 657.586816] env[64020]: DEBUG nova.network.neutron [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 657.587257] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg efa7ffbda73848c4b44d1259945abf15 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 657.594021] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg efa7ffbda73848c4b44d1259945abf15 [ 657.600993] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b6cc4b2b9995461080a29a60a45c6645 [ 658.088548] env[64020]: DEBUG nova.network.neutron [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 658.118205] env[64020]: DEBUG nova.network.neutron [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 658.197147] env[64020]: DEBUG nova.network.neutron [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 658.197674] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 3238e2b9288d49038e4f3e15e70c047f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 658.207454] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3238e2b9288d49038e4f3e15e70c047f [ 658.244757] env[64020]: DEBUG nova.network.neutron [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 658.245169] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 9a8054c8f59846c989abba7d9c2edb0d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 658.255853] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9a8054c8f59846c989abba7d9c2edb0d [ 658.626654] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2f106b8-9bcb-4d9c-baa6-b4f43d1a5c97 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.634847] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f76de1e-7361-4a86-a664-7b8efdfcb63c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.673348] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a97e313e-f6a4-4742-959b-5245b976b880 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.681865] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e6b4c5a-c003-4b27-9d5b-e456cd42607d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.697020] env[64020]: DEBUG nova.compute.provider_tree [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 658.697676] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg c336e3411e62455fa464920c82d87a4d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 658.699160] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Releasing lock "refresh_cache-65be7d88-0c12-4ad3-adc7-d089a1b91c28" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 658.699355] env[64020]: DEBUG nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 658.699523] env[64020]: DEBUG nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 658.699678] env[64020]: DEBUG nova.network.neutron [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 658.704743] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c336e3411e62455fa464920c82d87a4d [ 658.747240] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Releasing lock "refresh_cache-9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 658.747686] env[64020]: DEBUG nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 658.747872] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 658.748309] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0f279d81-4c6d-4c75-8f5b-bca5bff10b34 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.759491] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2e437f4-2ad6-463b-9c80-bca854f2f311 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.773669] env[64020]: DEBUG nova.network.neutron [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 658.774369] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 1cdcb98fcd1e4545b540f45415ac51b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 658.796372] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e could not be found. [ 658.796642] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 658.796837] env[64020]: INFO nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Took 0.05 seconds to destroy the instance on the hypervisor. [ 658.797093] env[64020]: DEBUG oslo.service.loopingcall [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 658.797795] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1cdcb98fcd1e4545b540f45415ac51b3 [ 658.799895] env[64020]: DEBUG nova.compute.manager [-] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 658.799895] env[64020]: DEBUG nova.network.neutron [-] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 658.826684] env[64020]: DEBUG nova.network.neutron [-] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 658.827387] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 76363ab0cc584f64831ec5ea99633075 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 658.834681] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 76363ab0cc584f64831ec5ea99633075 [ 659.200775] env[64020]: DEBUG nova.scheduler.client.report [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 659.204663] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 61688e3a304e48eca996af7c01713f31 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 659.216193] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 61688e3a304e48eca996af7c01713f31 [ 659.276099] env[64020]: DEBUG nova.network.neutron [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 659.276627] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 151ee075670842f4b702ab7f44b8af0b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 659.284769] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 151ee075670842f4b702ab7f44b8af0b [ 659.328948] env[64020]: DEBUG nova.network.neutron [-] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 659.329409] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 61aa9f78d5c44766bdfcda0778804b8d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 659.339025] env[64020]: DEBUG nova.compute.manager [req-1d649ac0-b804-46aa-ae80-734848b8eb57 req-12fe4789-cc6d-4d69-830d-5977c6dd6637 service nova] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Received event network-changed-89bd3d47-7a39-459a-bf81-51dab50c2f84 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 659.339247] env[64020]: DEBUG nova.compute.manager [req-1d649ac0-b804-46aa-ae80-734848b8eb57 req-12fe4789-cc6d-4d69-830d-5977c6dd6637 service nova] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Refreshing instance network info cache due to event network-changed-89bd3d47-7a39-459a-bf81-51dab50c2f84. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 659.339407] env[64020]: DEBUG oslo_concurrency.lockutils [req-1d649ac0-b804-46aa-ae80-734848b8eb57 req-12fe4789-cc6d-4d69-830d-5977c6dd6637 service nova] Acquiring lock "refresh_cache-9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 659.339621] env[64020]: DEBUG oslo_concurrency.lockutils [req-1d649ac0-b804-46aa-ae80-734848b8eb57 req-12fe4789-cc6d-4d69-830d-5977c6dd6637 service nova] Acquired lock "refresh_cache-9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 659.339796] env[64020]: DEBUG nova.network.neutron [req-1d649ac0-b804-46aa-ae80-734848b8eb57 req-12fe4789-cc6d-4d69-830d-5977c6dd6637 service nova] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Refreshing network info cache for port 89bd3d47-7a39-459a-bf81-51dab50c2f84 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 659.340224] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-1d649ac0-b804-46aa-ae80-734848b8eb57 req-12fe4789-cc6d-4d69-830d-5977c6dd6637 service nova] Expecting reply to msg d09320b1922f417b95ea82bee06e0808 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 659.341476] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 61aa9f78d5c44766bdfcda0778804b8d [ 659.346688] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d09320b1922f417b95ea82bee06e0808 [ 659.715524] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.167s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 659.715524] env[64020]: ERROR nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 5cc36a52-7d80-43b2-bc8b-0cb6be1a5090, please check neutron logs for more information. [ 659.715524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Traceback (most recent call last): [ 659.715524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 659.715524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] self.driver.spawn(context, instance, image_meta, [ 659.715524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 659.715524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 659.715524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 659.715524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] vm_ref = self.build_virtual_machine(instance, [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] vif_infos = vmwarevif.get_vif_info(self._session, [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] for vif in network_info: [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] return self._sync_wrapper(fn, *args, **kwargs) [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] self.wait() [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] self[:] = self._gt.wait() [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] return self._exit_event.wait() [ 659.717524] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] result = hub.switch() [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] return self.greenlet.switch() [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] result = function(*args, **kwargs) [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] return func(*args, **kwargs) [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] raise e [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] nwinfo = self.network_api.allocate_for_instance( [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 659.717958] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] created_port_ids = self._update_ports_for_instance( [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] with excutils.save_and_reraise_exception(): [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] self.force_reraise() [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] raise self.value [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] updated_port = self._update_port( [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] _ensure_no_port_binding_failure(port) [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 659.718423] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] raise exception.PortBindingFailed(port_id=port['id']) [ 659.718825] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] nova.exception.PortBindingFailed: Binding failed for port 5cc36a52-7d80-43b2-bc8b-0cb6be1a5090, please check neutron logs for more information. [ 659.718825] env[64020]: ERROR nova.compute.manager [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] [ 659.718825] env[64020]: DEBUG nova.compute.utils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Binding failed for port 5cc36a52-7d80-43b2-bc8b-0cb6be1a5090, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 659.718825] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.805s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 659.726624] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 45b2bc8ebe9b4303bdbe469a6728bd69 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 659.733552] env[64020]: DEBUG nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Build of instance e5b12149-0e4c-41b2-b726-f8a55452669a was re-scheduled: Binding failed for port 5cc36a52-7d80-43b2-bc8b-0cb6be1a5090, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 659.733552] env[64020]: DEBUG nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 659.733552] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquiring lock "refresh_cache-e5b12149-0e4c-41b2-b726-f8a55452669a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 659.733552] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Acquired lock "refresh_cache-e5b12149-0e4c-41b2-b726-f8a55452669a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 659.734207] env[64020]: DEBUG nova.network.neutron [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 659.734622] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 3699866c6c5e477db05767e79e1c618c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 659.744619] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3699866c6c5e477db05767e79e1c618c [ 659.763646] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 45b2bc8ebe9b4303bdbe469a6728bd69 [ 659.778665] env[64020]: INFO nova.compute.manager [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: 65be7d88-0c12-4ad3-adc7-d089a1b91c28] Took 1.08 seconds to deallocate network for instance. [ 659.780732] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg f19e83b13a2e428eb9f4c8614bcb7070 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 659.826088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f19e83b13a2e428eb9f4c8614bcb7070 [ 659.831585] env[64020]: INFO nova.compute.manager [-] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Took 1.03 seconds to deallocate network for instance. [ 659.850427] env[64020]: DEBUG nova.compute.claims [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 659.850427] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 659.876313] env[64020]: DEBUG nova.network.neutron [req-1d649ac0-b804-46aa-ae80-734848b8eb57 req-12fe4789-cc6d-4d69-830d-5977c6dd6637 service nova] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 659.999723] env[64020]: DEBUG nova.network.neutron [req-1d649ac0-b804-46aa-ae80-734848b8eb57 req-12fe4789-cc6d-4d69-830d-5977c6dd6637 service nova] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 660.000265] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-1d649ac0-b804-46aa-ae80-734848b8eb57 req-12fe4789-cc6d-4d69-830d-5977c6dd6637 service nova] Expecting reply to msg fa45e8350c5c42f4bf8f1f0caacc8466 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 660.011179] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fa45e8350c5c42f4bf8f1f0caacc8466 [ 660.286434] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 065df565620c4b84bf4813b159d85c4e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 660.292486] env[64020]: DEBUG nova.network.neutron [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 660.332946] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 065df565620c4b84bf4813b159d85c4e [ 660.481500] env[64020]: DEBUG nova.network.neutron [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 660.481500] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg eb37610ead6e4cae8cd1fa4357b981aa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 660.489304] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb37610ead6e4cae8cd1fa4357b981aa [ 660.502656] env[64020]: DEBUG oslo_concurrency.lockutils [req-1d649ac0-b804-46aa-ae80-734848b8eb57 req-12fe4789-cc6d-4d69-830d-5977c6dd6637 service nova] Releasing lock "refresh_cache-9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 660.503191] env[64020]: DEBUG nova.compute.manager [req-1d649ac0-b804-46aa-ae80-734848b8eb57 req-12fe4789-cc6d-4d69-830d-5977c6dd6637 service nova] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Received event network-vif-deleted-89bd3d47-7a39-459a-bf81-51dab50c2f84 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 660.804490] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e7f2116-60c7-43ff-b533-10ada98cbaff {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.814661] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f74b3d5-82b7-4c00-9f6e-f8e6feac4ea6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.820106] env[64020]: INFO nova.scheduler.client.report [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Deleted allocations for instance 65be7d88-0c12-4ad3-adc7-d089a1b91c28 [ 660.861371] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg f31fad17368d487393cedab1aef1d5f2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 660.867140] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54ebbd73-836e-470a-a566-218c164654ae {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.881133] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f31fad17368d487393cedab1aef1d5f2 [ 660.882751] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ce7fc96-decb-455c-a245-342a18dc9421 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.901427] env[64020]: DEBUG nova.compute.provider_tree [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 660.902131] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 87bd60fbdf5e48fc8be06b817f02dc33 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 660.918785] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 87bd60fbdf5e48fc8be06b817f02dc33 [ 660.983191] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Releasing lock "refresh_cache-e5b12149-0e4c-41b2-b726-f8a55452669a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 660.983483] env[64020]: DEBUG nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 660.983668] env[64020]: DEBUG nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 660.983834] env[64020]: DEBUG nova.network.neutron [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 661.009213] env[64020]: DEBUG nova.network.neutron [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 661.009831] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 57482e3f1f304382bac38e01b99b8d6d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 661.027887] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 57482e3f1f304382bac38e01b99b8d6d [ 661.374774] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1ed76b44-6c38-4954-aac5-06554bb1a329 tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "65be7d88-0c12-4ad3-adc7-d089a1b91c28" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 67.435s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 661.375394] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg 9e57ec494268430d8fdbfe663c6cde89 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 661.390811] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e57ec494268430d8fdbfe663c6cde89 [ 661.404855] env[64020]: DEBUG nova.scheduler.client.report [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 661.407359] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 4c4041687d7248989bd26735b8aebe88 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 661.421921] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c4041687d7248989bd26735b8aebe88 [ 661.511991] env[64020]: DEBUG nova.network.neutron [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 661.512582] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg c9bb26f57e104a1eb196ad01c9ad609d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 661.520668] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c9bb26f57e104a1eb196ad01c9ad609d [ 661.888875] env[64020]: DEBUG nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 661.888875] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg 73e4a1745c974fcb92921a761bea2ea3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 661.910906] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.196s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 661.911618] env[64020]: ERROR nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1382c442-7ac3-48fc-b680-81c4ea9727f2, please check neutron logs for more information. [ 661.911618] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Traceback (most recent call last): [ 661.911618] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 661.911618] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] self.driver.spawn(context, instance, image_meta, [ 661.911618] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 661.911618] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 661.911618] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 661.911618] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] vm_ref = self.build_virtual_machine(instance, [ 661.911618] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 661.911618] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] vif_infos = vmwarevif.get_vif_info(self._session, [ 661.911618] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] for vif in network_info: [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] return self._sync_wrapper(fn, *args, **kwargs) [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] self.wait() [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] self[:] = self._gt.wait() [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] return self._exit_event.wait() [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] result = hub.switch() [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 661.911902] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] return self.greenlet.switch() [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] result = function(*args, **kwargs) [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] return func(*args, **kwargs) [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] raise e [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] nwinfo = self.network_api.allocate_for_instance( [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] created_port_ids = self._update_ports_for_instance( [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] with excutils.save_and_reraise_exception(): [ 661.912257] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 661.912667] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] self.force_reraise() [ 661.912667] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 661.912667] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] raise self.value [ 661.912667] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 661.912667] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] updated_port = self._update_port( [ 661.912667] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 661.912667] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] _ensure_no_port_binding_failure(port) [ 661.912667] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 661.912667] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] raise exception.PortBindingFailed(port_id=port['id']) [ 661.912667] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] nova.exception.PortBindingFailed: Binding failed for port 1382c442-7ac3-48fc-b680-81c4ea9727f2, please check neutron logs for more information. [ 661.912667] env[64020]: ERROR nova.compute.manager [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] [ 661.912926] env[64020]: DEBUG nova.compute.utils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Binding failed for port 1382c442-7ac3-48fc-b680-81c4ea9727f2, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 661.913651] env[64020]: DEBUG oslo_concurrency.lockutils [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 18.061s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 661.913825] env[64020]: DEBUG nova.objects.instance [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Trying to apply a migration context that does not seem to be set for this instance {{(pid=64020) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 661.915297] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Expecting reply to msg d84f5cd9967d49e0ab46c68dc080aae1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 661.918016] env[64020]: DEBUG nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Build of instance a98ba289-b065-4cef-a7bd-7b7cd07b141d was re-scheduled: Binding failed for port 1382c442-7ac3-48fc-b680-81c4ea9727f2, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 661.919015] env[64020]: DEBUG nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 661.919656] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Acquiring lock "refresh_cache-a98ba289-b065-4cef-a7bd-7b7cd07b141d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 661.919656] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Acquired lock "refresh_cache-a98ba289-b065-4cef-a7bd-7b7cd07b141d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 661.919656] env[64020]: DEBUG nova.network.neutron [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 661.924035] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 9166f0600e0744e488dc3b675b3761f9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 661.932714] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9166f0600e0744e488dc3b675b3761f9 [ 661.950269] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 73e4a1745c974fcb92921a761bea2ea3 [ 662.011497] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d84f5cd9967d49e0ab46c68dc080aae1 [ 662.015849] env[64020]: INFO nova.compute.manager [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] [instance: e5b12149-0e4c-41b2-b726-f8a55452669a] Took 1.03 seconds to deallocate network for instance. [ 662.017218] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg ae89f6b3cf4b46269d6a0f8c26460027 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 662.056746] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae89f6b3cf4b46269d6a0f8c26460027 [ 662.400247] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 662.423568] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Expecting reply to msg b08e3615581f4f729981c2aa2950afea in queue reply_57893177120949e6a93cb88e15cd42b4 [ 662.430757] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b08e3615581f4f729981c2aa2950afea [ 662.521988] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 9169e476cdc1434bbfd8d36cae5759b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 662.523692] env[64020]: DEBUG nova.network.neutron [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 662.558206] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9169e476cdc1434bbfd8d36cae5759b3 [ 662.652590] env[64020]: DEBUG nova.network.neutron [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 662.653131] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 936e7212f68446c084a0064df5188207 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 662.662009] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 936e7212f68446c084a0064df5188207 [ 662.928182] env[64020]: DEBUG oslo_concurrency.lockutils [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.014s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 662.928590] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-85ca50f9-13b0-4f9b-838d-d3ed57b06a14 tempest-ServersAdmin275Test-1764604846 tempest-ServersAdmin275Test-1764604846-project-admin] Expecting reply to msg b173a59ae8244c659fbadab651b5e5e0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 662.929447] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.359s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 662.931223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg b39640a0e1b840f6bb52010e0c0ff90f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 662.943111] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b173a59ae8244c659fbadab651b5e5e0 [ 662.972491] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b39640a0e1b840f6bb52010e0c0ff90f [ 663.050402] env[64020]: INFO nova.scheduler.client.report [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Deleted allocations for instance e5b12149-0e4c-41b2-b726-f8a55452669a [ 663.056199] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Expecting reply to msg 5e7a0d57668b49e4928dfebe4887b3a6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 663.081407] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e7a0d57668b49e4928dfebe4887b3a6 [ 663.155182] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Releasing lock "refresh_cache-a98ba289-b065-4cef-a7bd-7b7cd07b141d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 663.155431] env[64020]: DEBUG nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 663.155592] env[64020]: DEBUG nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 663.155758] env[64020]: DEBUG nova.network.neutron [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 663.228134] env[64020]: DEBUG nova.network.neutron [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 663.228808] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 9205e56738a8464dba27d9e20bac368e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 663.237953] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9205e56738a8464dba27d9e20bac368e [ 663.566105] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b67cb19-8770-49a5-9d1d-675a1b2abb7a tempest-ServersAdminTestJSON-1231658706 tempest-ServersAdminTestJSON-1231658706-project-member] Lock "e5b12149-0e4c-41b2-b726-f8a55452669a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 67.316s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 663.566692] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 3e4aa04fda4545389954a587c0e78587 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 663.582325] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e4aa04fda4545389954a587c0e78587 [ 663.730514] env[64020]: DEBUG nova.network.neutron [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 663.730991] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg bd2a68be51aa4ee8857d5eb84442a240 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 663.742480] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bd2a68be51aa4ee8857d5eb84442a240 [ 663.874129] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3742d97-b2e3-4827-a0c5-bb76537bb69f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 663.883159] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f321495-065c-4b94-973d-892f5b5a4dd7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 663.914135] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5aa6936-5661-4902-aaba-239e020e81e9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 663.922430] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3808c77-8adc-47d5-8ffc-f681578a6ddb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 663.937082] env[64020]: DEBUG nova.compute.provider_tree [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 663.937082] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 59e7797232d444d7af7445cabaf4e63e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 663.944572] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59e7797232d444d7af7445cabaf4e63e [ 664.076386] env[64020]: DEBUG nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 664.076386] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg e9b188aa6c1f41138d5244d1280d985f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 664.109051] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9b188aa6c1f41138d5244d1280d985f [ 664.234169] env[64020]: INFO nova.compute.manager [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] [instance: a98ba289-b065-4cef-a7bd-7b7cd07b141d] Took 1.08 seconds to deallocate network for instance. [ 664.235192] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 1180d2d555cd4978b2a800c91715bba9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 664.273124] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1180d2d555cd4978b2a800c91715bba9 [ 664.443958] env[64020]: DEBUG nova.scheduler.client.report [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 664.443958] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg f5bb3ce5fcc64585b8079727e1af8925 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 664.456740] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5bb3ce5fcc64585b8079727e1af8925 [ 664.595048] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.739592] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg 9474d051ce744f0190c2d70042c3206f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 664.781364] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9474d051ce744f0190c2d70042c3206f [ 664.945994] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.016s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.946670] env[64020]: ERROR nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 860ce71a-5c1b-48d2-8d62-3545e5714ce4, please check neutron logs for more information. [ 664.946670] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Traceback (most recent call last): [ 664.946670] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 664.946670] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] self.driver.spawn(context, instance, image_meta, [ 664.946670] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 664.946670] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 664.946670] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 664.946670] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] vm_ref = self.build_virtual_machine(instance, [ 664.946670] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 664.946670] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] vif_infos = vmwarevif.get_vif_info(self._session, [ 664.946670] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] for vif in network_info: [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] return self._sync_wrapper(fn, *args, **kwargs) [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] self.wait() [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] self[:] = self._gt.wait() [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] return self._exit_event.wait() [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] result = hub.switch() [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 664.947076] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] return self.greenlet.switch() [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] result = function(*args, **kwargs) [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] return func(*args, **kwargs) [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] raise e [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] nwinfo = self.network_api.allocate_for_instance( [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] created_port_ids = self._update_ports_for_instance( [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] with excutils.save_and_reraise_exception(): [ 664.947358] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 664.947654] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] self.force_reraise() [ 664.947654] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 664.947654] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] raise self.value [ 664.947654] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 664.947654] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] updated_port = self._update_port( [ 664.947654] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 664.947654] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] _ensure_no_port_binding_failure(port) [ 664.947654] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 664.947654] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] raise exception.PortBindingFailed(port_id=port['id']) [ 664.947654] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] nova.exception.PortBindingFailed: Binding failed for port 860ce71a-5c1b-48d2-8d62-3545e5714ce4, please check neutron logs for more information. [ 664.947654] env[64020]: ERROR nova.compute.manager [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] [ 664.947901] env[64020]: DEBUG nova.compute.utils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Binding failed for port 860ce71a-5c1b-48d2-8d62-3545e5714ce4, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 664.952165] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.915s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.952165] env[64020]: INFO nova.compute.claims [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 664.952375] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 02f8adf0259c4b938be66800e25961bd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 664.953679] env[64020]: DEBUG nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Build of instance ae57ce0e-46b5-4490-af3f-4b2537bf316a was re-scheduled: Binding failed for port 860ce71a-5c1b-48d2-8d62-3545e5714ce4, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 664.954120] env[64020]: DEBUG nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 664.954349] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquiring lock "refresh_cache-ae57ce0e-46b5-4490-af3f-4b2537bf316a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 664.954524] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquired lock "refresh_cache-ae57ce0e-46b5-4490-af3f-4b2537bf316a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 664.954672] env[64020]: DEBUG nova.network.neutron [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 664.955037] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 63857d585dc14b9590ec9443628e638f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 664.962005] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 63857d585dc14b9590ec9443628e638f [ 664.994690] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02f8adf0259c4b938be66800e25961bd [ 665.110547] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Acquiring lock "5ccefb28-25a8-4876-a31c-58c7edfc9570" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 665.110765] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Lock "5ccefb28-25a8-4876-a31c-58c7edfc9570" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 665.267818] env[64020]: INFO nova.scheduler.client.report [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Deleted allocations for instance a98ba289-b065-4cef-a7bd-7b7cd07b141d [ 665.273668] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Expecting reply to msg d5f5e6e9e37a4e2baeee1ecadbbd2966 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 665.285466] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5f5e6e9e37a4e2baeee1ecadbbd2966 [ 665.458317] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg d1d52317161342eda0d69239d0d8a8c4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 665.469605] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1d52317161342eda0d69239d0d8a8c4 [ 665.480817] env[64020]: DEBUG nova.network.neutron [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 665.613762] env[64020]: DEBUG nova.network.neutron [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 665.614274] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 5525b946b677403f9c7ef90b8a720bf9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 665.622888] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5525b946b677403f9c7ef90b8a720bf9 [ 665.775743] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d8cef7ed-1323-4baf-a6cd-ce5a395d9c6c tempest-ServersAdminNegativeTestJSON-279935289 tempest-ServersAdminNegativeTestJSON-279935289-project-member] Lock "a98ba289-b065-4cef-a7bd-7b7cd07b141d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 68.699s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 665.776403] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg e6a360a6eccc458ebacc58e6f9fcce8f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 665.789895] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6a360a6eccc458ebacc58e6f9fcce8f [ 666.118399] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Releasing lock "refresh_cache-ae57ce0e-46b5-4490-af3f-4b2537bf316a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 666.118661] env[64020]: DEBUG nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 666.119056] env[64020]: DEBUG nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 666.119298] env[64020]: DEBUG nova.network.neutron [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 666.145619] env[64020]: DEBUG nova.network.neutron [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 666.145619] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 5b6b323adb1041ebab22d62b4f5484db in queue reply_57893177120949e6a93cb88e15cd42b4 [ 666.152894] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b6b323adb1041ebab22d62b4f5484db [ 666.279543] env[64020]: DEBUG nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 666.281650] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 6054b4bea8b140e5b6e7ec55d14e1418 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 666.327538] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6054b4bea8b140e5b6e7ec55d14e1418 [ 666.512890] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d16485e-3d43-459c-a42b-5165b6319c2f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 666.521493] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-740dbf23-1a15-4c12-8c4c-5320b1c42071 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 666.552950] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47d9e27e-18c1-4a24-b1bb-3c75bff5b1cd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 666.561179] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97ab08f0-8a0b-4377-9e52-b32bb851f62a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 666.584038] env[64020]: DEBUG nova.compute.provider_tree [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 666.584682] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg a20ebd8b30844b9082decb5a7bb24676 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 666.592344] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a20ebd8b30844b9082decb5a7bb24676 [ 666.647681] env[64020]: DEBUG nova.network.neutron [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 666.648388] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 17cc18f180fc490e966023012eef244d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 666.659473] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 17cc18f180fc490e966023012eef244d [ 666.809764] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 667.087509] env[64020]: DEBUG nova.scheduler.client.report [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 667.090181] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 7e4572df08254c9ea2b60e2f86d39858 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 667.108578] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7e4572df08254c9ea2b60e2f86d39858 [ 667.151144] env[64020]: INFO nova.compute.manager [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: ae57ce0e-46b5-4490-af3f-4b2537bf316a] Took 1.03 seconds to deallocate network for instance. [ 667.153038] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg b79515e688c649b3bff1ae8f4346f190 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 667.193839] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b79515e688c649b3bff1ae8f4346f190 [ 667.487094] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c0f953d69fe04af186b9577b7ef6e8c5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 667.499781] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0f953d69fe04af186b9577b7ef6e8c5 [ 667.608256] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.653s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 667.608256] env[64020]: DEBUG nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 667.608256] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg fafeac3046a942dd8aad3656d35b681a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 667.608256] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.755s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 667.608256] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 751420a95d3a4fc9a5f5253632abb0eb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 667.641628] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fafeac3046a942dd8aad3656d35b681a [ 667.649343] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 751420a95d3a4fc9a5f5253632abb0eb [ 667.657790] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 05f5d187acd044308ae3999a087eef47 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 667.692890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05f5d187acd044308ae3999a087eef47 [ 668.119734] env[64020]: DEBUG nova.compute.utils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 668.120422] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 24b4a643bae74b1a972572ecc591ca56 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 668.130189] env[64020]: DEBUG nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 668.130391] env[64020]: DEBUG nova.network.neutron [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 668.137149] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 24b4a643bae74b1a972572ecc591ca56 [ 668.185433] env[64020]: INFO nova.scheduler.client.report [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Deleted allocations for instance ae57ce0e-46b5-4490-af3f-4b2537bf316a [ 668.199545] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 6761963469cf493dbc94d3990c7f0c77 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 668.209939] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6761963469cf493dbc94d3990c7f0c77 [ 668.278229] env[64020]: DEBUG nova.policy [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '09307fac28464e3ea1c0af8333828824', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '03f27bf2042c44e0abba5ba9480749be', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 668.621034] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-542097fe-6f1c-4700-9609-7adc74ace8eb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.625379] env[64020]: DEBUG nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 668.627359] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 2d9f492dada64971946716cf0ec90024 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 668.631237] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8335a5f1-5ab7-48ca-9a92-0dce37c666a3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.661380] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-549bb030-419f-4bf6-b14e-2b6ab1f5b5a5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.665859] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d9f492dada64971946716cf0ec90024 [ 668.676177] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-677e648b-734b-4892-a622-3795175aaf02 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.687786] env[64020]: DEBUG nova.compute.provider_tree [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 668.688269] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg ce1c04597b3343e886e8a21164559ac3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 668.695511] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce1c04597b3343e886e8a21164559ac3 [ 668.699349] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9c7f196f-8f0c-4558-a13f-8446e628ab55 tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "ae57ce0e-46b5-4490-af3f-4b2537bf316a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 69.667s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 668.700058] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg c5bf37fb6aa24eebb51c6394dce50c1b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 668.709399] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5bf37fb6aa24eebb51c6394dce50c1b [ 669.133863] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 24a0a374579449d694ecc93978325e7d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 669.174563] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 24a0a374579449d694ecc93978325e7d [ 669.190894] env[64020]: DEBUG nova.scheduler.client.report [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 669.193424] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 84cef6893bd14bcf8e4c8e30d46005f2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 669.201964] env[64020]: DEBUG nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 669.203633] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg f5f23387d54b41e59063c60e5919c3b0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 669.209280] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 84cef6893bd14bcf8e4c8e30d46005f2 [ 669.258504] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5f23387d54b41e59063c60e5919c3b0 [ 669.342694] env[64020]: DEBUG nova.network.neutron [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Successfully created port: f5295483-d7ae-46c3-bef5-4edfb2f1ec32 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 669.637391] env[64020]: DEBUG nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 669.664739] env[64020]: DEBUG nova.virt.hardware [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:26:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='610004827',id=29,is_public=True,memory_mb=192,name='tempest-flavor_with_ephemeral_0-1290802617',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 669.665048] env[64020]: DEBUG nova.virt.hardware [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 669.665248] env[64020]: DEBUG nova.virt.hardware [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 669.665474] env[64020]: DEBUG nova.virt.hardware [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 669.665658] env[64020]: DEBUG nova.virt.hardware [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 669.665837] env[64020]: DEBUG nova.virt.hardware [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 669.666082] env[64020]: DEBUG nova.virt.hardware [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 669.666284] env[64020]: DEBUG nova.virt.hardware [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 669.666489] env[64020]: DEBUG nova.virt.hardware [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 669.666686] env[64020]: DEBUG nova.virt.hardware [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 669.666903] env[64020]: DEBUG nova.virt.hardware [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 669.667780] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0246b3f-cfb4-4ef2-8eec-715151fe4e65 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 669.676669] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e467a956-3c1c-4fc1-b1a4-4b2813cc1977 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 669.696946] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.091s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 669.697651] env[64020]: ERROR nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a872bec0-e59f-4427-a749-c89656be45fa, please check neutron logs for more information. [ 669.697651] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Traceback (most recent call last): [ 669.697651] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 669.697651] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] self.driver.spawn(context, instance, image_meta, [ 669.697651] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 669.697651] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] self._vmops.spawn(context, instance, image_meta, injected_files, [ 669.697651] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 669.697651] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] vm_ref = self.build_virtual_machine(instance, [ 669.697651] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 669.697651] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] vif_infos = vmwarevif.get_vif_info(self._session, [ 669.697651] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] for vif in network_info: [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] return self._sync_wrapper(fn, *args, **kwargs) [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] self.wait() [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] self[:] = self._gt.wait() [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] return self._exit_event.wait() [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] result = hub.switch() [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 669.697949] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] return self.greenlet.switch() [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] result = function(*args, **kwargs) [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] return func(*args, **kwargs) [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] raise e [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] nwinfo = self.network_api.allocate_for_instance( [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] created_port_ids = self._update_ports_for_instance( [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] with excutils.save_and_reraise_exception(): [ 669.698256] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 669.698558] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] self.force_reraise() [ 669.698558] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 669.698558] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] raise self.value [ 669.698558] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 669.698558] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] updated_port = self._update_port( [ 669.698558] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 669.698558] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] _ensure_no_port_binding_failure(port) [ 669.698558] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 669.698558] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] raise exception.PortBindingFailed(port_id=port['id']) [ 669.698558] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] nova.exception.PortBindingFailed: Binding failed for port a872bec0-e59f-4427-a749-c89656be45fa, please check neutron logs for more information. [ 669.698558] env[64020]: ERROR nova.compute.manager [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] [ 669.699053] env[64020]: DEBUG nova.compute.utils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Binding failed for port a872bec0-e59f-4427-a749-c89656be45fa, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 669.700333] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 18.770s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 669.700687] env[64020]: DEBUG nova.objects.instance [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lazy-loading 'resources' on Instance uuid b3440b55-2469-46fa-ac2c-3e207bf530ec {{(pid=64020) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 669.701137] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 1d8d376bf0b04102bd873f1f7f1dddf0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 669.702287] env[64020]: DEBUG nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Build of instance 76de4647-9281-4c49-b4f1-16f0ea2ff313 was re-scheduled: Binding failed for port a872bec0-e59f-4427-a749-c89656be45fa, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 669.703049] env[64020]: DEBUG nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 669.703557] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquiring lock "refresh_cache-76de4647-9281-4c49-b4f1-16f0ea2ff313" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 669.703796] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Acquired lock "refresh_cache-76de4647-9281-4c49-b4f1-16f0ea2ff313" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 669.704021] env[64020]: DEBUG nova.network.neutron [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 669.704593] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg c2a7f0a915c14c809e0057399f5ee1ba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 669.715317] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2a7f0a915c14c809e0057399f5ee1ba [ 669.716294] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d8d376bf0b04102bd873f1f7f1dddf0 [ 669.737684] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 669.749362] env[64020]: DEBUG nova.network.neutron [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 669.860091] env[64020]: DEBUG nova.network.neutron [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 669.861051] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg cd2efc55920443bdbdbef16e5bf4b75f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 669.871026] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd2efc55920443bdbdbef16e5bf4b75f [ 670.161883] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-372587a4-0309-4a92-88be-a674a870819f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 670.170861] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa8368d4-00bf-427e-8455-b29a058f340a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 670.205857] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de3928be-c8c7-445b-9dc0-96160e61bb0f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 670.214334] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-617bd9f9-7713-46b7-bdd0-88fe273c5f04 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 670.231803] env[64020]: DEBUG nova.compute.provider_tree [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 670.232480] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 23c435cedad941aa89504cde9debc58f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 670.240455] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 23c435cedad941aa89504cde9debc58f [ 670.366549] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Releasing lock "refresh_cache-76de4647-9281-4c49-b4f1-16f0ea2ff313" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 670.366549] env[64020]: DEBUG nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 670.366549] env[64020]: DEBUG nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 670.366549] env[64020]: DEBUG nova.network.neutron [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 670.399946] env[64020]: DEBUG nova.network.neutron [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 670.399946] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg c993ad1438b74eacb2987e86eeaf17b6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 670.408318] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c993ad1438b74eacb2987e86eeaf17b6 [ 670.735705] env[64020]: DEBUG nova.scheduler.client.report [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 670.738675] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg fbb1d73c74dd4f0097ec3f9cbb270431 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 670.752363] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fbb1d73c74dd4f0097ec3f9cbb270431 [ 670.901720] env[64020]: DEBUG nova.network.neutron [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 670.902219] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 833d6a608dcb46d182ab4c36050df1d5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 670.912546] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 833d6a608dcb46d182ab4c36050df1d5 [ 671.246640] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.541s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 671.246640] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.780s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 671.246640] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg 34e73c8411914032a0ebb4622666bc4d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 671.277851] env[64020]: INFO nova.scheduler.client.report [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Deleted allocations for instance b3440b55-2469-46fa-ac2c-3e207bf530ec [ 671.282329] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg b988c37a67aa466195a5d9efce42a30d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 671.310514] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 34e73c8411914032a0ebb4622666bc4d [ 671.341717] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b988c37a67aa466195a5d9efce42a30d [ 671.404411] env[64020]: INFO nova.compute.manager [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] [instance: 76de4647-9281-4c49-b4f1-16f0ea2ff313] Took 1.04 seconds to deallocate network for instance. [ 671.406301] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 3880b0606c2140ad8db4e07c8f9f3602 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 671.447944] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3880b0606c2140ad8db4e07c8f9f3602 [ 671.561466] env[64020]: DEBUG nova.compute.manager [req-ad68d6c4-4b8a-4456-97ff-b24404292fc7 req-3a8d11bb-5494-435d-b07f-f88abcedbaa1 service nova] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Received event network-changed-f5295483-d7ae-46c3-bef5-4edfb2f1ec32 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 671.561639] env[64020]: DEBUG nova.compute.manager [req-ad68d6c4-4b8a-4456-97ff-b24404292fc7 req-3a8d11bb-5494-435d-b07f-f88abcedbaa1 service nova] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Refreshing instance network info cache due to event network-changed-f5295483-d7ae-46c3-bef5-4edfb2f1ec32. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 671.561847] env[64020]: DEBUG oslo_concurrency.lockutils [req-ad68d6c4-4b8a-4456-97ff-b24404292fc7 req-3a8d11bb-5494-435d-b07f-f88abcedbaa1 service nova] Acquiring lock "refresh_cache-cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 671.561983] env[64020]: DEBUG oslo_concurrency.lockutils [req-ad68d6c4-4b8a-4456-97ff-b24404292fc7 req-3a8d11bb-5494-435d-b07f-f88abcedbaa1 service nova] Acquired lock "refresh_cache-cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 671.562147] env[64020]: DEBUG nova.network.neutron [req-ad68d6c4-4b8a-4456-97ff-b24404292fc7 req-3a8d11bb-5494-435d-b07f-f88abcedbaa1 service nova] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Refreshing network info cache for port f5295483-d7ae-46c3-bef5-4edfb2f1ec32 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 671.562570] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-ad68d6c4-4b8a-4456-97ff-b24404292fc7 req-3a8d11bb-5494-435d-b07f-f88abcedbaa1 service nova] Expecting reply to msg 972d1284c6e54beb8de4b2622860abb4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 671.572163] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 972d1284c6e54beb8de4b2622860abb4 [ 671.787068] env[64020]: ERROR nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f5295483-d7ae-46c3-bef5-4edfb2f1ec32, please check neutron logs for more information. [ 671.787068] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 671.787068] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 671.787068] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 671.787068] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 671.787068] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 671.787068] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 671.787068] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 671.787068] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 671.787068] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 671.787068] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 671.787068] env[64020]: ERROR nova.compute.manager raise self.value [ 671.787068] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 671.787068] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 671.787068] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 671.787068] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 671.787451] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 671.787451] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 671.787451] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f5295483-d7ae-46c3-bef5-4edfb2f1ec32, please check neutron logs for more information. [ 671.787451] env[64020]: ERROR nova.compute.manager [ 671.787451] env[64020]: Traceback (most recent call last): [ 671.787451] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 671.787451] env[64020]: listener.cb(fileno) [ 671.787451] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 671.787451] env[64020]: result = function(*args, **kwargs) [ 671.787451] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 671.787451] env[64020]: return func(*args, **kwargs) [ 671.787451] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 671.787451] env[64020]: raise e [ 671.787451] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 671.787451] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 671.787451] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 671.787451] env[64020]: created_port_ids = self._update_ports_for_instance( [ 671.787451] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 671.787451] env[64020]: with excutils.save_and_reraise_exception(): [ 671.787451] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 671.787451] env[64020]: self.force_reraise() [ 671.787451] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 671.787451] env[64020]: raise self.value [ 671.787451] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 671.787451] env[64020]: updated_port = self._update_port( [ 671.787451] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 671.787451] env[64020]: _ensure_no_port_binding_failure(port) [ 671.787451] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 671.787451] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 671.788186] env[64020]: nova.exception.PortBindingFailed: Binding failed for port f5295483-d7ae-46c3-bef5-4edfb2f1ec32, please check neutron logs for more information. [ 671.788186] env[64020]: Removing descriptor: 16 [ 671.788186] env[64020]: ERROR nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f5295483-d7ae-46c3-bef5-4edfb2f1ec32, please check neutron logs for more information. [ 671.788186] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Traceback (most recent call last): [ 671.788186] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 671.788186] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] yield resources [ 671.788186] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 671.788186] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] self.driver.spawn(context, instance, image_meta, [ 671.788186] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 671.788186] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 671.788186] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 671.788186] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] vm_ref = self.build_virtual_machine(instance, [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] vif_infos = vmwarevif.get_vif_info(self._session, [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] for vif in network_info: [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] return self._sync_wrapper(fn, *args, **kwargs) [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] self.wait() [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] self[:] = self._gt.wait() [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] return self._exit_event.wait() [ 671.788528] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] result = hub.switch() [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] return self.greenlet.switch() [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] result = function(*args, **kwargs) [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] return func(*args, **kwargs) [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] raise e [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] nwinfo = self.network_api.allocate_for_instance( [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 671.788829] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] created_port_ids = self._update_ports_for_instance( [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] with excutils.save_and_reraise_exception(): [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] self.force_reraise() [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] raise self.value [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] updated_port = self._update_port( [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] _ensure_no_port_binding_failure(port) [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 671.789188] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] raise exception.PortBindingFailed(port_id=port['id']) [ 671.789465] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] nova.exception.PortBindingFailed: Binding failed for port f5295483-d7ae-46c3-bef5-4edfb2f1ec32, please check neutron logs for more information. [ 671.789465] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] [ 671.789465] env[64020]: INFO nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Terminating instance [ 671.790499] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Acquiring lock "refresh_cache-cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 671.794219] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Lock "b3440b55-2469-46fa-ac2c-3e207bf530ec" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 24.573s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 671.794219] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f497a277-7abb-48e9-b648-ad7ed697f037 tempest-ServersAdmin275Test-1358236042 tempest-ServersAdmin275Test-1358236042-project-member] Expecting reply to msg 55c847bf29fe43a3b0c911f8d616432d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 671.807167] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 55c847bf29fe43a3b0c911f8d616432d [ 671.911272] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 6d0bbb44a8134f8bbdc1a8ce7b71eb74 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 671.970207] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d0bbb44a8134f8bbdc1a8ce7b71eb74 [ 672.107053] env[64020]: DEBUG nova.network.neutron [req-ad68d6c4-4b8a-4456-97ff-b24404292fc7 req-3a8d11bb-5494-435d-b07f-f88abcedbaa1 service nova] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 672.256112] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e26ee8a-68f4-4ccf-b1ed-aa12ec556a4a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 672.265397] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c6fb8c8-d65e-47c7-8d0b-e01e98a0d09b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 672.300890] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5096e165-121c-4068-b94f-923b9d18571f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 672.311525] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0485eb35-b34f-4bfd-beb5-cf9aa11d2dc3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 672.326519] env[64020]: DEBUG nova.compute.provider_tree [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 672.327038] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg e0df9919678d4e5c90cb052b65627021 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 672.350833] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e0df9919678d4e5c90cb052b65627021 [ 672.437449] env[64020]: INFO nova.scheduler.client.report [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Deleted allocations for instance 76de4647-9281-4c49-b4f1-16f0ea2ff313 [ 672.444327] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Expecting reply to msg 543b0b846cc940adbec56f597128286f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 672.459394] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 543b0b846cc940adbec56f597128286f [ 672.591113] env[64020]: DEBUG nova.network.neutron [req-ad68d6c4-4b8a-4456-97ff-b24404292fc7 req-3a8d11bb-5494-435d-b07f-f88abcedbaa1 service nova] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 672.591676] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-ad68d6c4-4b8a-4456-97ff-b24404292fc7 req-3a8d11bb-5494-435d-b07f-f88abcedbaa1 service nova] Expecting reply to msg a94cef6dcad24eb39eccc69f025e0b99 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 672.606440] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a94cef6dcad24eb39eccc69f025e0b99 [ 672.832607] env[64020]: DEBUG nova.scheduler.client.report [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 672.832607] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg bd9f94d2de83404db908d70338b6c830 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 672.850354] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bd9f94d2de83404db908d70338b6c830 [ 672.947113] env[64020]: DEBUG oslo_concurrency.lockutils [None req-12ac3ba4-6a18-49bb-a14d-ad0adeb2e63d tempest-ListImageFiltersTestJSON-515464580 tempest-ListImageFiltersTestJSON-515464580-project-member] Lock "76de4647-9281-4c49-b4f1-16f0ea2ff313" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 72.571s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 672.947748] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 88e0a5e397dc45b5866d0e025383e072 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 672.962171] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88e0a5e397dc45b5866d0e025383e072 [ 673.094895] env[64020]: DEBUG oslo_concurrency.lockutils [req-ad68d6c4-4b8a-4456-97ff-b24404292fc7 req-3a8d11bb-5494-435d-b07f-f88abcedbaa1 service nova] Releasing lock "refresh_cache-cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 673.095309] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Acquired lock "refresh_cache-cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 673.095491] env[64020]: DEBUG nova.network.neutron [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 673.095938] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 362103168daf4bf8bbbd396356fad10c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 673.106903] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 362103168daf4bf8bbbd396356fad10c [ 673.334517] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.091s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 673.335149] env[64020]: ERROR nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 59e1b8cf-9c47-43ad-8b36-bef6d9a79790, please check neutron logs for more information. [ 673.335149] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Traceback (most recent call last): [ 673.335149] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 673.335149] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] self.driver.spawn(context, instance, image_meta, [ 673.335149] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 673.335149] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 673.335149] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 673.335149] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] vm_ref = self.build_virtual_machine(instance, [ 673.335149] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 673.335149] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] vif_infos = vmwarevif.get_vif_info(self._session, [ 673.335149] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] for vif in network_info: [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] return self._sync_wrapper(fn, *args, **kwargs) [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] self.wait() [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] self[:] = self._gt.wait() [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] return self._exit_event.wait() [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] result = hub.switch() [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 673.335429] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] return self.greenlet.switch() [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] result = function(*args, **kwargs) [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] return func(*args, **kwargs) [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] raise e [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] nwinfo = self.network_api.allocate_for_instance( [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] created_port_ids = self._update_ports_for_instance( [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] with excutils.save_and_reraise_exception(): [ 673.335716] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 673.335997] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] self.force_reraise() [ 673.335997] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 673.335997] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] raise self.value [ 673.335997] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 673.335997] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] updated_port = self._update_port( [ 673.335997] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 673.335997] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] _ensure_no_port_binding_failure(port) [ 673.335997] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 673.335997] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] raise exception.PortBindingFailed(port_id=port['id']) [ 673.335997] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] nova.exception.PortBindingFailed: Binding failed for port 59e1b8cf-9c47-43ad-8b36-bef6d9a79790, please check neutron logs for more information. [ 673.335997] env[64020]: ERROR nova.compute.manager [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] [ 673.336255] env[64020]: DEBUG nova.compute.utils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Binding failed for port 59e1b8cf-9c47-43ad-8b36-bef6d9a79790, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 673.337127] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.631s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 673.339223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg eac6b2bfee864b7d82d01f6a0a5bd73f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 673.344650] env[64020]: DEBUG nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Build of instance bfb84a86-8de0-4b2d-9355-85e0d8b5eba2 was re-scheduled: Binding failed for port 59e1b8cf-9c47-43ad-8b36-bef6d9a79790, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 673.345101] env[64020]: DEBUG nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 673.345326] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Acquiring lock "refresh_cache-bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 673.345463] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Acquired lock "refresh_cache-bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 673.345612] env[64020]: DEBUG nova.network.neutron [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 673.346036] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg 49159bf4e8474ffa9d30ee22614d1536 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 673.353018] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 49159bf4e8474ffa9d30ee22614d1536 [ 673.397884] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eac6b2bfee864b7d82d01f6a0a5bd73f [ 673.449989] env[64020]: DEBUG nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 673.451775] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 0f8c83c2d9954f93991c61c5bca75135 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 673.483847] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f8c83c2d9954f93991c61c5bca75135 [ 673.628542] env[64020]: DEBUG nova.network.neutron [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 673.785636] env[64020]: DEBUG nova.network.neutron [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 673.785636] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 79d3c93c7ee74b128d3c65f4f7b06bc7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 673.794715] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 79d3c93c7ee74b128d3c65f4f7b06bc7 [ 673.836756] env[64020]: DEBUG nova.compute.manager [req-80067da8-9b49-498e-b1cd-73b29e5a201b req-aa4b16fb-d7c0-4b2f-a3a5-8fc362a48ba8 service nova] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Received event network-vif-deleted-f5295483-d7ae-46c3-bef5-4edfb2f1ec32 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 673.880307] env[64020]: DEBUG nova.network.neutron [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 673.973176] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 674.162891] env[64020]: DEBUG nova.network.neutron [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 674.162891] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg 4e503a31229646ec80742743b0b334ee in queue reply_57893177120949e6a93cb88e15cd42b4 [ 674.175281] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e503a31229646ec80742743b0b334ee [ 674.289451] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Releasing lock "refresh_cache-cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 674.289451] env[64020]: DEBUG nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 674.289571] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 674.289791] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8b2d99e9-6268-45b6-bcf9-706a93f6424d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 674.299388] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff24dc57-e42c-4cb6-bfd1-2254c18b30f8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 674.332181] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance cea1a347-7f1a-4090-b3ad-54ccfbdfea2a could not be found. [ 674.332445] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 674.332693] env[64020]: INFO nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Took 0.04 seconds to destroy the instance on the hypervisor. [ 674.332954] env[64020]: DEBUG oslo.service.loopingcall [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 674.333420] env[64020]: DEBUG nova.compute.manager [-] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 674.333530] env[64020]: DEBUG nova.network.neutron [-] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 674.343855] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05fcd9bf-b5f5-4a3f-a80f-1ee7e6dfb241 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 674.358346] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af486191-2d9e-48b1-8c06-cc4d6b079cc4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 674.394372] env[64020]: DEBUG nova.network.neutron [-] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 674.394906] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6c491f1eece548628527cbdb7025114c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 674.396253] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ae9ae10-d20f-4497-9810-62ed00304021 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 674.404174] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6c491f1eece548628527cbdb7025114c [ 674.405900] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-193babcc-367a-40a2-8d50-3197a54deee0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 674.423049] env[64020]: DEBUG nova.compute.provider_tree [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 674.423604] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 4090e37654f943e69d8cb0f86d9674a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 674.430852] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4090e37654f943e69d8cb0f86d9674a2 [ 674.665393] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Releasing lock "refresh_cache-bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 674.665665] env[64020]: DEBUG nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 674.665762] env[64020]: DEBUG nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 674.665981] env[64020]: DEBUG nova.network.neutron [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 674.714037] env[64020]: DEBUG nova.network.neutron [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 674.714722] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg 6fb4d2d99e094959bad2dc3f9f6c662f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 674.721219] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6fb4d2d99e094959bad2dc3f9f6c662f [ 674.819248] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Acquiring lock "a39b40c8-0340-4778-8b43-f29ed51664bc" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 674.819479] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Lock "a39b40c8-0340-4778-8b43-f29ed51664bc" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 674.899768] env[64020]: DEBUG nova.network.neutron [-] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 674.900279] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg d41f072f9cea44ca8958337d97a93752 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 674.909318] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d41f072f9cea44ca8958337d97a93752 [ 674.925992] env[64020]: DEBUG nova.scheduler.client.report [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 674.928431] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 90672023f8bc41628216f183f2ef389c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 674.943412] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90672023f8bc41628216f183f2ef389c [ 675.224496] env[64020]: DEBUG nova.network.neutron [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 675.224496] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg dfeaafe6147448f0824a9312e0b14881 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 675.227229] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dfeaafe6147448f0824a9312e0b14881 [ 675.406661] env[64020]: INFO nova.compute.manager [-] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Took 1.07 seconds to deallocate network for instance. [ 675.408221] env[64020]: DEBUG nova.compute.claims [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 675.408221] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 675.436021] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.095s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 675.436021] env[64020]: ERROR nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 2d332da1-175a-4a4b-8822-5e571fb24234, please check neutron logs for more information. [ 675.436021] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Traceback (most recent call last): [ 675.436021] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 675.436021] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] self.driver.spawn(context, instance, image_meta, [ 675.436021] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 675.436021] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] self._vmops.spawn(context, instance, image_meta, injected_files, [ 675.436021] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 675.436021] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] vm_ref = self.build_virtual_machine(instance, [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] vif_infos = vmwarevif.get_vif_info(self._session, [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] for vif in network_info: [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] return self._sync_wrapper(fn, *args, **kwargs) [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] self.wait() [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] self[:] = self._gt.wait() [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] return self._exit_event.wait() [ 675.436309] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] result = hub.switch() [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] return self.greenlet.switch() [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] result = function(*args, **kwargs) [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] return func(*args, **kwargs) [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] raise e [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] nwinfo = self.network_api.allocate_for_instance( [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 675.436598] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] created_port_ids = self._update_ports_for_instance( [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] with excutils.save_and_reraise_exception(): [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] self.force_reraise() [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] raise self.value [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] updated_port = self._update_port( [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] _ensure_no_port_binding_failure(port) [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 675.436888] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] raise exception.PortBindingFailed(port_id=port['id']) [ 675.437166] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] nova.exception.PortBindingFailed: Binding failed for port 2d332da1-175a-4a4b-8822-5e571fb24234, please check neutron logs for more information. [ 675.437166] env[64020]: ERROR nova.compute.manager [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] [ 675.437166] env[64020]: DEBUG nova.compute.utils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Binding failed for port 2d332da1-175a-4a4b-8822-5e571fb24234, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 675.437166] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.544s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 675.438670] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 41e0f934e8c44c7087b74c0410a281f7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 675.447849] env[64020]: DEBUG nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Build of instance 618c3b06-1fd8-45d3-9c59-61c0e202a299 was re-scheduled: Binding failed for port 2d332da1-175a-4a4b-8822-5e571fb24234, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 675.447849] env[64020]: DEBUG nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 675.447849] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Acquiring lock "refresh_cache-618c3b06-1fd8-45d3-9c59-61c0e202a299" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 675.447849] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Acquired lock "refresh_cache-618c3b06-1fd8-45d3-9c59-61c0e202a299" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 675.447995] env[64020]: DEBUG nova.network.neutron [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 675.447995] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 3cf390cde5214b6cb54d4c73b16bab65 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 675.454310] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3cf390cde5214b6cb54d4c73b16bab65 [ 675.492300] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 41e0f934e8c44c7087b74c0410a281f7 [ 675.726195] env[64020]: INFO nova.compute.manager [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] [instance: bfb84a86-8de0-4b2d-9355-85e0d8b5eba2] Took 1.06 seconds to deallocate network for instance. [ 675.727769] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg c5b492c594c14b6eb5e379d1ab343e0a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 675.765637] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5b492c594c14b6eb5e379d1ab343e0a [ 675.970192] env[64020]: DEBUG nova.network.neutron [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 676.232450] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg 8d08265b74ed4ef2a7de2d567215d1e8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 676.270380] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d08265b74ed4ef2a7de2d567215d1e8 [ 676.271436] env[64020]: DEBUG nova.network.neutron [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 676.271900] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 917511c5edb849bfb67ae3bdcf1a8fab in queue reply_57893177120949e6a93cb88e15cd42b4 [ 676.280564] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 917511c5edb849bfb67ae3bdcf1a8fab [ 676.400086] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4753799c-c644-4496-b44c-6f58f3be3d07 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 676.408204] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9774e519-fc55-44fb-b7e4-692bf4f0e657 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 676.438591] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-369d0924-f1f4-47af-9ad6-6f8d61cad1f0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 676.447573] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a1a0303-ee9a-46b0-a142-3dd0632957d5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 676.466216] env[64020]: DEBUG nova.compute.provider_tree [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 676.466729] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 9bf590c730594dac836c383331380aed in queue reply_57893177120949e6a93cb88e15cd42b4 [ 676.476350] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9bf590c730594dac836c383331380aed [ 676.755849] env[64020]: INFO nova.scheduler.client.report [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Deleted allocations for instance bfb84a86-8de0-4b2d-9355-85e0d8b5eba2 [ 676.765491] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Expecting reply to msg 185f59451d3d48c39369fd409778a7ad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 676.777532] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Releasing lock "refresh_cache-618c3b06-1fd8-45d3-9c59-61c0e202a299" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 676.777532] env[64020]: DEBUG nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 676.777532] env[64020]: DEBUG nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 676.777532] env[64020]: DEBUG nova.network.neutron [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 676.794775] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 185f59451d3d48c39369fd409778a7ad [ 676.805805] env[64020]: DEBUG nova.network.neutron [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 676.806681] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 618f4d5c1b3449cc8c113490d90b0815 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 676.821261] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 618f4d5c1b3449cc8c113490d90b0815 [ 676.969572] env[64020]: DEBUG nova.scheduler.client.report [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 676.972103] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 09960fa937294430a010370f9adfd621 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 676.994714] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 09960fa937294430a010370f9adfd621 [ 677.274200] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1eda50f-a35f-4724-90a1-f4b41d5aa103 tempest-VolumesAssistedSnapshotsTest-881392271 tempest-VolumesAssistedSnapshotsTest-881392271-project-member] Lock "bfb84a86-8de0-4b2d-9355-85e0d8b5eba2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 75.601s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 677.274833] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg d7ef8a7d03dc47e4950eb2a372d7e0f3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 677.289223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7ef8a7d03dc47e4950eb2a372d7e0f3 [ 677.308471] env[64020]: DEBUG nova.network.neutron [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 677.308971] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg e5cc0d2f6b2e40aa846c22690cc34335 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 677.322434] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e5cc0d2f6b2e40aa846c22690cc34335 [ 677.474794] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.040s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 677.475400] env[64020]: ERROR nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a1866c0a-948c-477e-b20f-1d7959b22ab6, please check neutron logs for more information. [ 677.475400] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Traceback (most recent call last): [ 677.475400] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 677.475400] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] self.driver.spawn(context, instance, image_meta, [ 677.475400] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 677.475400] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] self._vmops.spawn(context, instance, image_meta, injected_files, [ 677.475400] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 677.475400] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] vm_ref = self.build_virtual_machine(instance, [ 677.475400] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 677.475400] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] vif_infos = vmwarevif.get_vif_info(self._session, [ 677.475400] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] for vif in network_info: [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] return self._sync_wrapper(fn, *args, **kwargs) [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] self.wait() [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] self[:] = self._gt.wait() [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] return self._exit_event.wait() [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] result = hub.switch() [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 677.475745] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] return self.greenlet.switch() [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] result = function(*args, **kwargs) [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] return func(*args, **kwargs) [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] raise e [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] nwinfo = self.network_api.allocate_for_instance( [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] created_port_ids = self._update_ports_for_instance( [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] with excutils.save_and_reraise_exception(): [ 677.476081] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 677.476387] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] self.force_reraise() [ 677.476387] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 677.476387] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] raise self.value [ 677.476387] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 677.476387] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] updated_port = self._update_port( [ 677.476387] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 677.476387] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] _ensure_no_port_binding_failure(port) [ 677.476387] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 677.476387] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] raise exception.PortBindingFailed(port_id=port['id']) [ 677.476387] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] nova.exception.PortBindingFailed: Binding failed for port a1866c0a-948c-477e-b20f-1d7959b22ab6, please check neutron logs for more information. [ 677.476387] env[64020]: ERROR nova.compute.manager [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] [ 677.476643] env[64020]: DEBUG nova.compute.utils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Binding failed for port a1866c0a-948c-477e-b20f-1d7959b22ab6, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 677.477473] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.628s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 677.479228] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 76b09445ee9f4f6a90b5de49f39fcbda in queue reply_57893177120949e6a93cb88e15cd42b4 [ 677.481464] env[64020]: DEBUG nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Build of instance 4478746a-11a6-480d-84c1-c6f04ab33505 was re-scheduled: Binding failed for port a1866c0a-948c-477e-b20f-1d7959b22ab6, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 677.482005] env[64020]: DEBUG nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 677.482341] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Acquiring lock "refresh_cache-4478746a-11a6-480d-84c1-c6f04ab33505" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 677.482403] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Acquired lock "refresh_cache-4478746a-11a6-480d-84c1-c6f04ab33505" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 677.482568] env[64020]: DEBUG nova.network.neutron [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 677.482916] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 06d62f6b623a4456b3e20d5f449fbf69 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 677.487572] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Acquiring lock "73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 677.487768] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Lock "73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 677.489242] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06d62f6b623a4456b3e20d5f449fbf69 [ 677.515559] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 76b09445ee9f4f6a90b5de49f39fcbda [ 677.777107] env[64020]: DEBUG nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 677.778911] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg ee7e88f6b6dd45a782a838de51c985a4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 677.811788] env[64020]: INFO nova.compute.manager [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] [instance: 618c3b06-1fd8-45d3-9c59-61c0e202a299] Took 1.03 seconds to deallocate network for instance. [ 677.813836] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 4dd6aa9329074ac29f9c71a23a6162c2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 677.829531] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ee7e88f6b6dd45a782a838de51c985a4 [ 677.864640] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4dd6aa9329074ac29f9c71a23a6162c2 [ 678.121562] env[64020]: DEBUG nova.network.neutron [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 678.297308] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 678.325726] env[64020]: DEBUG nova.network.neutron [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 678.326451] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg d6a9ad1d10984327b3ca37f913b7d4b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 678.333453] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg fe0201cb596849879e4e8d70001a5cf2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 678.347988] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d6a9ad1d10984327b3ca37f913b7d4b3 [ 678.424747] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe0201cb596849879e4e8d70001a5cf2 [ 678.504720] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff003ed5-e297-4a1f-9e96-f2bd9e1f5b68 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 678.513281] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c0e77960-3602-4397-a146-fcfd11d975c5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 678.550735] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af1bd36c-6c00-4804-b4b8-796532e931f1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 678.559535] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73f5a98a-d4a4-4a80-8b31-c69b863716d1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 679.228798] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Releasing lock "refresh_cache-4478746a-11a6-480d-84c1-c6f04ab33505" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 679.229030] env[64020]: DEBUG nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 679.229220] env[64020]: DEBUG nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 679.229370] env[64020]: DEBUG nova.network.neutron [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 679.244778] env[64020]: DEBUG nova.compute.provider_tree [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 679.245170] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg e4b04d287c004a7ca3adfce3bcc220a1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 679.258241] env[64020]: INFO nova.scheduler.client.report [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Deleted allocations for instance 618c3b06-1fd8-45d3-9c59-61c0e202a299 [ 679.263262] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e4b04d287c004a7ca3adfce3bcc220a1 [ 679.264464] env[64020]: DEBUG nova.network.neutron [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 679.265035] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 5b965fa933fb44cfaded75010627e3a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 679.266300] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Expecting reply to msg 171ea0665a374921bbaf6c9276d6d6d3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 679.274405] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b965fa933fb44cfaded75010627e3a2 [ 679.301579] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 171ea0665a374921bbaf6c9276d6d6d3 [ 679.748296] env[64020]: DEBUG nova.scheduler.client.report [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 679.750793] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 0faea93ef0f8496c87be6a2811003524 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 679.766543] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0faea93ef0f8496c87be6a2811003524 [ 679.770777] env[64020]: DEBUG oslo_concurrency.lockutils [None req-18731398-9a19-4bcf-b92c-0e67dc02a1d0 tempest-ServersV294TestFqdnHostnames-1016909389 tempest-ServersV294TestFqdnHostnames-1016909389-project-member] Lock "618c3b06-1fd8-45d3-9c59-61c0e202a299" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 75.753s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 679.771072] env[64020]: DEBUG nova.network.neutron [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 679.771590] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg fb5d7f04dfe04f9e9f9fafb537210f24 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 679.773071] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg be1baa28c23948a9984bd5df0b8f3ed4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 679.782850] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb5d7f04dfe04f9e9f9fafb537210f24 [ 679.783324] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be1baa28c23948a9984bd5df0b8f3ed4 [ 680.253749] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.776s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 680.254372] env[64020]: ERROR nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 89bd3d47-7a39-459a-bf81-51dab50c2f84, please check neutron logs for more information. [ 680.254372] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Traceback (most recent call last): [ 680.254372] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 680.254372] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] self.driver.spawn(context, instance, image_meta, [ 680.254372] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 680.254372] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 680.254372] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 680.254372] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] vm_ref = self.build_virtual_machine(instance, [ 680.254372] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 680.254372] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] vif_infos = vmwarevif.get_vif_info(self._session, [ 680.254372] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] for vif in network_info: [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] return self._sync_wrapper(fn, *args, **kwargs) [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] self.wait() [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] self[:] = self._gt.wait() [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] return self._exit_event.wait() [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] current.throw(*self._exc) [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 680.254656] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] result = function(*args, **kwargs) [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] return func(*args, **kwargs) [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] raise e [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] nwinfo = self.network_api.allocate_for_instance( [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] created_port_ids = self._update_ports_for_instance( [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] with excutils.save_and_reraise_exception(): [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] self.force_reraise() [ 680.254968] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 680.255424] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] raise self.value [ 680.255424] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 680.255424] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] updated_port = self._update_port( [ 680.255424] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 680.255424] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] _ensure_no_port_binding_failure(port) [ 680.255424] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 680.255424] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] raise exception.PortBindingFailed(port_id=port['id']) [ 680.255424] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] nova.exception.PortBindingFailed: Binding failed for port 89bd3d47-7a39-459a-bf81-51dab50c2f84, please check neutron logs for more information. [ 680.255424] env[64020]: ERROR nova.compute.manager [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] [ 680.255424] env[64020]: DEBUG nova.compute.utils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Binding failed for port 89bd3d47-7a39-459a-bf81-51dab50c2f84, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 680.256352] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.857s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 680.258136] env[64020]: INFO nova.compute.claims [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 680.260217] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg c813c0040072426789929f5e66b59f48 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 680.273164] env[64020]: DEBUG nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Build of instance 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e was re-scheduled: Binding failed for port 89bd3d47-7a39-459a-bf81-51dab50c2f84, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 680.273727] env[64020]: DEBUG nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 680.273987] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Acquiring lock "refresh_cache-9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 680.274145] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Acquired lock "refresh_cache-9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 680.274300] env[64020]: DEBUG nova.network.neutron [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 680.274731] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg e21440237bbc4af69cfe9f0384fa2797 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 680.275759] env[64020]: INFO nova.compute.manager [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] [instance: 4478746a-11a6-480d-84c1-c6f04ab33505] Took 1.05 seconds to deallocate network for instance. [ 680.277369] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 2be27fccd2894599a719e076b154c74a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 680.278329] env[64020]: DEBUG nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 680.279869] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg d6ddc3648cd34692a5c8b2fca9f5b608 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 680.296023] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e21440237bbc4af69cfe9f0384fa2797 [ 680.324408] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d6ddc3648cd34692a5c8b2fca9f5b608 [ 680.324968] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c813c0040072426789929f5e66b59f48 [ 680.335836] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2be27fccd2894599a719e076b154c74a [ 680.775529] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg e20f0e0e4d7344b1b3c355acae82d1b7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 680.786375] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 665de79a1df2473cb81607e021867273 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 680.790966] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e20f0e0e4d7344b1b3c355acae82d1b7 [ 680.808562] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 680.826088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 665de79a1df2473cb81607e021867273 [ 680.872735] env[64020]: DEBUG nova.network.neutron [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 681.108890] env[64020]: DEBUG nova.network.neutron [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 681.109606] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 0be95f6413c9411bbfd10c802d31d82d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 681.119144] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0be95f6413c9411bbfd10c802d31d82d [ 681.317690] env[64020]: INFO nova.scheduler.client.report [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Deleted allocations for instance 4478746a-11a6-480d-84c1-c6f04ab33505 [ 681.329520] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Expecting reply to msg 58ff7217d80c44c4ab8d68aff4514330 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 681.353149] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 58ff7217d80c44c4ab8d68aff4514330 [ 681.611752] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Releasing lock "refresh_cache-9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 681.611978] env[64020]: DEBUG nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 681.612215] env[64020]: DEBUG nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 681.612394] env[64020]: DEBUG nova.network.neutron [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 681.663099] env[64020]: DEBUG nova.network.neutron [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 681.663684] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 8dfb37a7cbec489195d240cb74b68111 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 681.672601] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8dfb37a7cbec489195d240cb74b68111 [ 681.704139] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72288edc-5bb8-4e0e-957f-e40812bee6b9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.714844] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee02d6c7-79c3-4ea9-b29f-277d64f9f93e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.745836] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3971ecae-fc41-496d-b6c4-233e8c0a04a5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.754228] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-efb4c20c-c793-4fbd-890f-8d2f46da5195 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.769447] env[64020]: DEBUG nova.compute.provider_tree [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 681.769974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg 608fa708fd1d4ce28c132e55664acf3d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 681.778908] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 608fa708fd1d4ce28c132e55664acf3d [ 681.831857] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1fe0c5f-dc2f-47b9-858d-9ae6136ad8a7 tempest-ServerRescueTestJSONUnderV235-191907024 tempest-ServerRescueTestJSONUnderV235-191907024-project-member] Lock "4478746a-11a6-480d-84c1-c6f04ab33505" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 76.273s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 681.832556] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg d42f81dfae844e9d8e1b31b298741a83 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 681.846361] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d42f81dfae844e9d8e1b31b298741a83 [ 682.005109] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquiring lock "d6228174-ceb1-4ca0-b62b-27db15c20a85" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 682.005346] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "d6228174-ceb1-4ca0-b62b-27db15c20a85" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 682.169920] env[64020]: DEBUG nova.network.neutron [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 682.170463] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 094dfaf22eb5449d80bd8239abd0498a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 682.180085] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 094dfaf22eb5449d80bd8239abd0498a [ 682.276465] env[64020]: DEBUG nova.scheduler.client.report [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 682.276465] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg d0a7a196bdc046129e840472c1a23293 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 682.288067] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0a7a196bdc046129e840472c1a23293 [ 682.336460] env[64020]: DEBUG nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 682.336766] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg a2f3fe17243b45679ae847d81daee5fa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 682.386640] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2f3fe17243b45679ae847d81daee5fa [ 682.566216] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 682.566427] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 682.567084] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 9da2bee31bf648d18a97d1a137a7cfb5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 682.577364] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9da2bee31bf648d18a97d1a137a7cfb5 [ 682.672839] env[64020]: INFO nova.compute.manager [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] [instance: 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e] Took 1.06 seconds to deallocate network for instance. [ 682.674882] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 9000e0a0386a49f5bfd84113300fb63c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 682.713261] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9000e0a0386a49f5bfd84113300fb63c [ 682.779469] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.523s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 682.779973] env[64020]: DEBUG nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 682.781746] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg b35fd92c88844148bb03489800395e32 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 682.783533] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.189s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 682.785568] env[64020]: INFO nova.compute.claims [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 682.787353] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 4a5a2194a5b14a948953e94341d32362 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 682.825308] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b35fd92c88844148bb03489800395e32 [ 682.837330] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4a5a2194a5b14a948953e94341d32362 [ 682.858589] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 683.075673] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 683.075931] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Starting heal instance info cache {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 683.075997] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Rebuilding the list of instances to heal {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 683.077836] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 298cde583bf74e39a2722000bdc3792d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 683.089088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 298cde583bf74e39a2722000bdc3792d [ 683.179804] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg cc1b3ec24f7e480cabd051fa0298bbb9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 683.214034] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cc1b3ec24f7e480cabd051fa0298bbb9 [ 683.285491] env[64020]: DEBUG nova.compute.utils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 683.286322] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg 02c3734df23840908460c11b3439bdc8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 683.287815] env[64020]: DEBUG nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 683.288013] env[64020]: DEBUG nova.network.neutron [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 683.292212] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg abc71cf98b084294bcbb659bc3e163bd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 683.300686] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abc71cf98b084294bcbb659bc3e163bd [ 683.301388] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02c3734df23840908460c11b3439bdc8 [ 683.451949] env[64020]: DEBUG nova.policy [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c6b0c7e9dafe4a108841db587ca55d90', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '48acd6014f78494384fd8940e91ba3fc', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 683.581111] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 683.581275] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 683.581407] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 683.581528] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Didn't find any instances for network info cache update. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 683.581725] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 683.581873] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 683.582011] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 683.582150] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 683.582352] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 683.582538] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 683.582661] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=64020) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 683.582802] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager.update_available_resource {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 683.583142] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg e70da5fa3b38428d8a484bf5ed62d690 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 683.594867] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e70da5fa3b38428d8a484bf5ed62d690 [ 683.706475] env[64020]: INFO nova.scheduler.client.report [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Deleted allocations for instance 9d6c361e-1dec-4e47-9d1e-728fce1c2c7e [ 683.713092] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Expecting reply to msg 6daae7ea8866476d9e44f2a63af1bc1d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 683.733083] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6daae7ea8866476d9e44f2a63af1bc1d [ 683.792120] env[64020]: DEBUG nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 683.794202] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg aabfff69f3764503bbfe8d7aa2ea88d4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 683.837598] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aabfff69f3764503bbfe8d7aa2ea88d4 [ 684.086411] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 684.224880] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4704052c-78cf-4d5f-af10-49d76f3598ca tempest-ServerActionsTestOtherB-1752236685 tempest-ServerActionsTestOtherB-1752236685-project-member] Lock "9d6c361e-1dec-4e47-9d1e-728fce1c2c7e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 77.983s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 684.225653] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg f69e8fa726894dce9b56ae462db47262 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 684.243629] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f69e8fa726894dce9b56ae462db47262 [ 684.276896] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5af5abf6-32e1-4dda-b2d4-9a5b4d1c431a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.285746] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f2bafbf-ea03-4286-a0fe-afbbc954365a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.324765] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg dc7547dc89ab4bf28a0114b676699f46 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 684.327333] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-449522bb-84bb-4473-b46b-b8971df66cdc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.337289] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c415ad9-682e-4de4-83ba-774b02c0887d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.353295] env[64020]: DEBUG nova.compute.provider_tree [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 684.353855] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 14b20b5f4ecc49cbb5dde32d335472c3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 684.361580] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14b20b5f4ecc49cbb5dde32d335472c3 [ 684.369945] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc7547dc89ab4bf28a0114b676699f46 [ 684.422397] env[64020]: DEBUG nova.network.neutron [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Successfully created port: e1f39557-dca1-4318-9e14-86de0b534ae7 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 684.730455] env[64020]: DEBUG nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 684.732361] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg d2c17ea2b17747ee841edcfae56ffc2e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 684.775339] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d2c17ea2b17747ee841edcfae56ffc2e [ 684.828480] env[64020]: DEBUG nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 684.856877] env[64020]: DEBUG nova.scheduler.client.report [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 684.859354] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 2d9b66010d204fc5bcdf7ed00241c6b4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 684.864368] env[64020]: DEBUG nova.virt.hardware [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 684.864368] env[64020]: DEBUG nova.virt.hardware [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 684.864368] env[64020]: DEBUG nova.virt.hardware [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 684.864513] env[64020]: DEBUG nova.virt.hardware [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 684.864513] env[64020]: DEBUG nova.virt.hardware [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 684.864513] env[64020]: DEBUG nova.virt.hardware [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 684.864513] env[64020]: DEBUG nova.virt.hardware [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 684.864513] env[64020]: DEBUG nova.virt.hardware [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 684.864644] env[64020]: DEBUG nova.virt.hardware [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 684.864644] env[64020]: DEBUG nova.virt.hardware [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 684.864700] env[64020]: DEBUG nova.virt.hardware [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 684.865656] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-302c93f4-7a36-403c-8b5e-c695a6e7d6ce {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.877877] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da628ea0-9b4a-4821-bf02-1c00758ccd3e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.885327] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d9b66010d204fc5bcdf7ed00241c6b4 [ 685.253709] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 685.369795] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.586s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.370300] env[64020]: DEBUG nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 685.372062] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg f632f8a885604f519434b03d7f75689e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 685.373109] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.564s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 685.374649] env[64020]: INFO nova.compute.claims [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 685.381449] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 74f18b3ccff44d96b7cfae3bec55deb9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 685.409255] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f632f8a885604f519434b03d7f75689e [ 685.418099] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74f18b3ccff44d96b7cfae3bec55deb9 [ 685.856407] env[64020]: DEBUG nova.compute.manager [req-f81f9329-381b-4db0-932a-47276c308736 req-0932b4a4-9432-4c76-9802-98a2ca28dc58 service nova] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Received event network-changed-e1f39557-dca1-4318-9e14-86de0b534ae7 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 685.856655] env[64020]: DEBUG nova.compute.manager [req-f81f9329-381b-4db0-932a-47276c308736 req-0932b4a4-9432-4c76-9802-98a2ca28dc58 service nova] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Refreshing instance network info cache due to event network-changed-e1f39557-dca1-4318-9e14-86de0b534ae7. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 685.856794] env[64020]: DEBUG oslo_concurrency.lockutils [req-f81f9329-381b-4db0-932a-47276c308736 req-0932b4a4-9432-4c76-9802-98a2ca28dc58 service nova] Acquiring lock "refresh_cache-5ac25e1d-007a-4eaa-811c-655b54efd7ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 685.856928] env[64020]: DEBUG oslo_concurrency.lockutils [req-f81f9329-381b-4db0-932a-47276c308736 req-0932b4a4-9432-4c76-9802-98a2ca28dc58 service nova] Acquired lock "refresh_cache-5ac25e1d-007a-4eaa-811c-655b54efd7ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 685.857084] env[64020]: DEBUG nova.network.neutron [req-f81f9329-381b-4db0-932a-47276c308736 req-0932b4a4-9432-4c76-9802-98a2ca28dc58 service nova] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Refreshing network info cache for port e1f39557-dca1-4318-9e14-86de0b534ae7 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 685.857574] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-f81f9329-381b-4db0-932a-47276c308736 req-0932b4a4-9432-4c76-9802-98a2ca28dc58 service nova] Expecting reply to msg ea95555b4bb5435986381be8fedec8b1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 685.864682] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ea95555b4bb5435986381be8fedec8b1 [ 685.883041] env[64020]: DEBUG nova.compute.utils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 685.883951] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 1691b0c010ab4d2982105f786eb23a13 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 685.886837] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg f3236462ab4b4a779f6c8bb03b5dd8cb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 685.887922] env[64020]: DEBUG nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 685.888376] env[64020]: DEBUG nova.network.neutron [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 685.896069] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3236462ab4b4a779f6c8bb03b5dd8cb [ 685.900486] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1691b0c010ab4d2982105f786eb23a13 [ 685.987401] env[64020]: DEBUG nova.policy [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02b993f5bbd5442a8d2c06cff095e867', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e0c4cb7c0e9941f7a056fef708be822c', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 686.214202] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquiring lock "e27f988a-a64b-407b-ad66-4bed7ee45b52" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 686.214463] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "e27f988a-a64b-407b-ad66-4bed7ee45b52" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 686.240444] env[64020]: ERROR nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e1f39557-dca1-4318-9e14-86de0b534ae7, please check neutron logs for more information. [ 686.240444] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 686.240444] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 686.240444] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 686.240444] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 686.240444] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 686.240444] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 686.240444] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 686.240444] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 686.240444] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 686.240444] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 686.240444] env[64020]: ERROR nova.compute.manager raise self.value [ 686.240444] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 686.240444] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 686.240444] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 686.240444] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 686.241012] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 686.241012] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 686.241012] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e1f39557-dca1-4318-9e14-86de0b534ae7, please check neutron logs for more information. [ 686.241012] env[64020]: ERROR nova.compute.manager [ 686.241012] env[64020]: Traceback (most recent call last): [ 686.241012] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 686.241012] env[64020]: listener.cb(fileno) [ 686.241012] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 686.241012] env[64020]: result = function(*args, **kwargs) [ 686.241012] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 686.241012] env[64020]: return func(*args, **kwargs) [ 686.241012] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 686.241012] env[64020]: raise e [ 686.241012] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 686.241012] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 686.241012] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 686.241012] env[64020]: created_port_ids = self._update_ports_for_instance( [ 686.241012] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 686.241012] env[64020]: with excutils.save_and_reraise_exception(): [ 686.241012] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 686.241012] env[64020]: self.force_reraise() [ 686.241012] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 686.241012] env[64020]: raise self.value [ 686.241012] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 686.241012] env[64020]: updated_port = self._update_port( [ 686.241012] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 686.241012] env[64020]: _ensure_no_port_binding_failure(port) [ 686.241012] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 686.241012] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 686.241788] env[64020]: nova.exception.PortBindingFailed: Binding failed for port e1f39557-dca1-4318-9e14-86de0b534ae7, please check neutron logs for more information. [ 686.241788] env[64020]: Removing descriptor: 16 [ 686.241788] env[64020]: ERROR nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e1f39557-dca1-4318-9e14-86de0b534ae7, please check neutron logs for more information. [ 686.241788] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Traceback (most recent call last): [ 686.241788] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 686.241788] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] yield resources [ 686.241788] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 686.241788] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] self.driver.spawn(context, instance, image_meta, [ 686.241788] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 686.241788] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] self._vmops.spawn(context, instance, image_meta, injected_files, [ 686.241788] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 686.241788] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] vm_ref = self.build_virtual_machine(instance, [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] vif_infos = vmwarevif.get_vif_info(self._session, [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] for vif in network_info: [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] return self._sync_wrapper(fn, *args, **kwargs) [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] self.wait() [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] self[:] = self._gt.wait() [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] return self._exit_event.wait() [ 686.242248] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] result = hub.switch() [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] return self.greenlet.switch() [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] result = function(*args, **kwargs) [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] return func(*args, **kwargs) [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] raise e [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] nwinfo = self.network_api.allocate_for_instance( [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 686.242546] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] created_port_ids = self._update_ports_for_instance( [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] with excutils.save_and_reraise_exception(): [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] self.force_reraise() [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] raise self.value [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] updated_port = self._update_port( [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] _ensure_no_port_binding_failure(port) [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 686.242918] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] raise exception.PortBindingFailed(port_id=port['id']) [ 686.243207] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] nova.exception.PortBindingFailed: Binding failed for port e1f39557-dca1-4318-9e14-86de0b534ae7, please check neutron logs for more information. [ 686.243207] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] [ 686.243207] env[64020]: INFO nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Terminating instance [ 686.243207] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Acquiring lock "refresh_cache-5ac25e1d-007a-4eaa-811c-655b54efd7ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 686.388850] env[64020]: DEBUG nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 686.390558] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg ea21f650e1d6497cb2723e98c0079c4d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 686.432893] env[64020]: DEBUG nova.network.neutron [req-f81f9329-381b-4db0-932a-47276c308736 req-0932b4a4-9432-4c76-9802-98a2ca28dc58 service nova] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 686.445169] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ea21f650e1d6497cb2723e98c0079c4d [ 686.450146] env[64020]: DEBUG nova.network.neutron [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Successfully created port: b7cef9b2-89e8-4bfc-8a6b-fe379e46900f {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 686.591432] env[64020]: DEBUG nova.network.neutron [req-f81f9329-381b-4db0-932a-47276c308736 req-0932b4a4-9432-4c76-9802-98a2ca28dc58 service nova] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 686.591947] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-f81f9329-381b-4db0-932a-47276c308736 req-0932b4a4-9432-4c76-9802-98a2ca28dc58 service nova] Expecting reply to msg e67fdf0b044e45b9aff93cb812241fd1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 686.600498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e67fdf0b044e45b9aff93cb812241fd1 [ 686.897643] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 3b3a5debe2bc4d228b129a9c647138bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 686.942043] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4ddc479-bae0-4f28-b081-0885ef6f6f74 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.947245] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b3a5debe2bc4d228b129a9c647138bc [ 686.953938] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d99263ab-a29d-49f5-aaf8-3a9a99a08d80 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.990160] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3db84628-d331-4f23-afff-59c85caa1550 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.999461] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ca54709-6398-4b46-a0e9-509ffa40a0f2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.014929] env[64020]: DEBUG nova.compute.provider_tree [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 687.015524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg a959611a26924ea5912670e1384e8793 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 687.023431] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a959611a26924ea5912670e1384e8793 [ 687.096231] env[64020]: DEBUG oslo_concurrency.lockutils [req-f81f9329-381b-4db0-932a-47276c308736 req-0932b4a4-9432-4c76-9802-98a2ca28dc58 service nova] Releasing lock "refresh_cache-5ac25e1d-007a-4eaa-811c-655b54efd7ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 687.096865] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Acquired lock "refresh_cache-5ac25e1d-007a-4eaa-811c-655b54efd7ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 687.097085] env[64020]: DEBUG nova.network.neutron [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 687.098186] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg a26a22278993415f81673c95563947cf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 687.107177] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a26a22278993415f81673c95563947cf [ 687.402830] env[64020]: DEBUG nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 687.424545] env[64020]: DEBUG nova.virt.hardware [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 687.424709] env[64020]: DEBUG nova.virt.hardware [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 687.424857] env[64020]: DEBUG nova.virt.hardware [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 687.425028] env[64020]: DEBUG nova.virt.hardware [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 687.425165] env[64020]: DEBUG nova.virt.hardware [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 687.425302] env[64020]: DEBUG nova.virt.hardware [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 687.425502] env[64020]: DEBUG nova.virt.hardware [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 687.425651] env[64020]: DEBUG nova.virt.hardware [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 687.425809] env[64020]: DEBUG nova.virt.hardware [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 687.425961] env[64020]: DEBUG nova.virt.hardware [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 687.426120] env[64020]: DEBUG nova.virt.hardware [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 687.427060] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45d41c6e-b69e-4f53-a1bd-50e0da7ace74 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.436614] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-323a7453-c15c-4625-b853-3e09f7a727d4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.518846] env[64020]: DEBUG nova.scheduler.client.report [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 687.521811] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg d2c38c02a581456ebdd5e7d037efef25 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 687.538538] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d2c38c02a581456ebdd5e7d037efef25 [ 687.622838] env[64020]: DEBUG nova.network.neutron [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 687.722577] env[64020]: DEBUG nova.network.neutron [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 687.723254] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg 62ac0a95e3b84877aa0fd6552ccf4c09 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 687.737298] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62ac0a95e3b84877aa0fd6552ccf4c09 [ 688.030524] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.656s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 688.030524] env[64020]: DEBUG nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 688.031961] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 674d2c8bc8384fd8a91848b35900df1f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 688.033795] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.296s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 688.041560] env[64020]: INFO nova.compute.claims [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 688.043184] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 71745452e28246a8818766197e13552d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 688.068508] env[64020]: DEBUG nova.compute.manager [req-d85c0f59-27da-46a0-a0f6-a0cb79dba644 req-4aaa7e5b-4ff4-41a4-9499-ac5f5f1613d7 service nova] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Received event network-vif-deleted-e1f39557-dca1-4318-9e14-86de0b534ae7 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 688.090534] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 674d2c8bc8384fd8a91848b35900df1f [ 688.091094] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 71745452e28246a8818766197e13552d [ 688.128979] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 688.129241] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 688.229452] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Releasing lock "refresh_cache-5ac25e1d-007a-4eaa-811c-655b54efd7ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 688.229452] env[64020]: DEBUG nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 688.229452] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 688.229452] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-53aa930d-cdb6-4366-887d-ea7b0389ffbb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 688.240290] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2e72156-1a3b-48b8-ae0f-2dcbe7acf68e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 688.266239] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5ac25e1d-007a-4eaa-811c-655b54efd7ec could not be found. [ 688.266597] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 688.266903] env[64020]: INFO nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Took 0.04 seconds to destroy the instance on the hypervisor. [ 688.267245] env[64020]: DEBUG oslo.service.loopingcall [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 688.267569] env[64020]: DEBUG nova.compute.manager [-] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 688.267744] env[64020]: DEBUG nova.network.neutron [-] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 688.286653] env[64020]: DEBUG nova.network.neutron [-] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 688.287329] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c3dc147e66ce448cae07419a9e76c09c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 688.294937] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3dc147e66ce448cae07419a9e76c09c [ 688.535633] env[64020]: DEBUG nova.compute.utils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 688.536169] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 0be6b4f52b0146279b36eda15df2f426 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 688.537493] env[64020]: DEBUG nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 688.537651] env[64020]: DEBUG nova.network.neutron [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 688.546413] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 9af250971c4d4806abeab34ae3f84017 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 688.548360] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0be6b4f52b0146279b36eda15df2f426 [ 688.553223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9af250971c4d4806abeab34ae3f84017 [ 688.624572] env[64020]: DEBUG nova.compute.manager [req-f6bf8748-f64a-4bd3-bb75-f4d0d3f72c0c req-d22b452b-4942-47b5-9e01-2b2204979447 service nova] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Received event network-changed-b7cef9b2-89e8-4bfc-8a6b-fe379e46900f {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 688.624744] env[64020]: DEBUG nova.compute.manager [req-f6bf8748-f64a-4bd3-bb75-f4d0d3f72c0c req-d22b452b-4942-47b5-9e01-2b2204979447 service nova] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Refreshing instance network info cache due to event network-changed-b7cef9b2-89e8-4bfc-8a6b-fe379e46900f. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 688.624957] env[64020]: DEBUG oslo_concurrency.lockutils [req-f6bf8748-f64a-4bd3-bb75-f4d0d3f72c0c req-d22b452b-4942-47b5-9e01-2b2204979447 service nova] Acquiring lock "refresh_cache-d4d0d50d-8479-47f7-ba29-114ef10764cc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 688.625094] env[64020]: DEBUG oslo_concurrency.lockutils [req-f6bf8748-f64a-4bd3-bb75-f4d0d3f72c0c req-d22b452b-4942-47b5-9e01-2b2204979447 service nova] Acquired lock "refresh_cache-d4d0d50d-8479-47f7-ba29-114ef10764cc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 688.625253] env[64020]: DEBUG nova.network.neutron [req-f6bf8748-f64a-4bd3-bb75-f4d0d3f72c0c req-d22b452b-4942-47b5-9e01-2b2204979447 service nova] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Refreshing network info cache for port b7cef9b2-89e8-4bfc-8a6b-fe379e46900f {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 688.626092] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-f6bf8748-f64a-4bd3-bb75-f4d0d3f72c0c req-d22b452b-4942-47b5-9e01-2b2204979447 service nova] Expecting reply to msg 3e1162d6687e4369a0a1d34d4488bd4f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 688.632598] env[64020]: DEBUG nova.policy [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '1c1610f2158f4501bda3ba14d5282882', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8da409b9607348ddb9edd6e6579e5f32', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 688.638883] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e1162d6687e4369a0a1d34d4488bd4f [ 688.789648] env[64020]: DEBUG nova.network.neutron [-] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 688.790131] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 456b77e6636044c3826c608d6897c58c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 688.798298] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 456b77e6636044c3826c608d6897c58c [ 688.863558] env[64020]: ERROR nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b7cef9b2-89e8-4bfc-8a6b-fe379e46900f, please check neutron logs for more information. [ 688.863558] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 688.863558] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 688.863558] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 688.863558] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 688.863558] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 688.863558] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 688.863558] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 688.863558] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 688.863558] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 688.863558] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 688.863558] env[64020]: ERROR nova.compute.manager raise self.value [ 688.863558] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 688.863558] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 688.863558] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 688.863558] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 688.864168] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 688.864168] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 688.864168] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b7cef9b2-89e8-4bfc-8a6b-fe379e46900f, please check neutron logs for more information. [ 688.864168] env[64020]: ERROR nova.compute.manager [ 688.864168] env[64020]: Traceback (most recent call last): [ 688.864168] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 688.864168] env[64020]: listener.cb(fileno) [ 688.864168] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 688.864168] env[64020]: result = function(*args, **kwargs) [ 688.864168] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 688.864168] env[64020]: return func(*args, **kwargs) [ 688.864168] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 688.864168] env[64020]: raise e [ 688.864168] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 688.864168] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 688.864168] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 688.864168] env[64020]: created_port_ids = self._update_ports_for_instance( [ 688.864168] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 688.864168] env[64020]: with excutils.save_and_reraise_exception(): [ 688.864168] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 688.864168] env[64020]: self.force_reraise() [ 688.864168] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 688.864168] env[64020]: raise self.value [ 688.864168] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 688.864168] env[64020]: updated_port = self._update_port( [ 688.864168] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 688.864168] env[64020]: _ensure_no_port_binding_failure(port) [ 688.864168] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 688.864168] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 688.865109] env[64020]: nova.exception.PortBindingFailed: Binding failed for port b7cef9b2-89e8-4bfc-8a6b-fe379e46900f, please check neutron logs for more information. [ 688.865109] env[64020]: Removing descriptor: 18 [ 688.865109] env[64020]: ERROR nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b7cef9b2-89e8-4bfc-8a6b-fe379e46900f, please check neutron logs for more information. [ 688.865109] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Traceback (most recent call last): [ 688.865109] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 688.865109] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] yield resources [ 688.865109] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 688.865109] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] self.driver.spawn(context, instance, image_meta, [ 688.865109] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 688.865109] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 688.865109] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 688.865109] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] vm_ref = self.build_virtual_machine(instance, [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] vif_infos = vmwarevif.get_vif_info(self._session, [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] for vif in network_info: [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] return self._sync_wrapper(fn, *args, **kwargs) [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] self.wait() [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] self[:] = self._gt.wait() [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] return self._exit_event.wait() [ 688.865555] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] result = hub.switch() [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] return self.greenlet.switch() [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] result = function(*args, **kwargs) [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] return func(*args, **kwargs) [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] raise e [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] nwinfo = self.network_api.allocate_for_instance( [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 688.865997] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] created_port_ids = self._update_ports_for_instance( [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] with excutils.save_and_reraise_exception(): [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] self.force_reraise() [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] raise self.value [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] updated_port = self._update_port( [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] _ensure_no_port_binding_failure(port) [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 688.866502] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] raise exception.PortBindingFailed(port_id=port['id']) [ 688.866895] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] nova.exception.PortBindingFailed: Binding failed for port b7cef9b2-89e8-4bfc-8a6b-fe379e46900f, please check neutron logs for more information. [ 688.866895] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] [ 688.866895] env[64020]: INFO nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Terminating instance [ 688.866895] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-d4d0d50d-8479-47f7-ba29-114ef10764cc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 689.042907] env[64020]: DEBUG nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 689.043343] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg abee0aedad104008af9e28e80cb5c73a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 689.084353] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abee0aedad104008af9e28e80cb5c73a [ 689.146106] env[64020]: DEBUG nova.network.neutron [req-f6bf8748-f64a-4bd3-bb75-f4d0d3f72c0c req-d22b452b-4942-47b5-9e01-2b2204979447 service nova] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 689.235670] env[64020]: DEBUG nova.network.neutron [req-f6bf8748-f64a-4bd3-bb75-f4d0d3f72c0c req-d22b452b-4942-47b5-9e01-2b2204979447 service nova] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 689.236196] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-f6bf8748-f64a-4bd3-bb75-f4d0d3f72c0c req-d22b452b-4942-47b5-9e01-2b2204979447 service nova] Expecting reply to msg 2dc3c052eb184079991495ead77046b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 689.244353] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2dc3c052eb184079991495ead77046b3 [ 689.291790] env[64020]: INFO nova.compute.manager [-] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Took 1.02 seconds to deallocate network for instance. [ 689.294133] env[64020]: DEBUG nova.compute.claims [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 689.294314] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 689.389027] env[64020]: DEBUG nova.network.neutron [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Successfully created port: 7d184f73-764c-4ec8-84ce-066b295da5fb {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 689.432834] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08d272cd-2f4c-4683-836d-cf31bb54df28 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 689.441250] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-872771a4-bc9f-400b-8a90-51a5d33b9915 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 689.481006] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36883b8e-f793-4dd6-b84f-8649bca4512d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 689.490745] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc9a8a4f-e6c5-4a46-af11-364f39152472 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 689.507448] env[64020]: DEBUG nova.compute.provider_tree [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 689.507974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg bdb9282c80c94f1dab41ba84cef337be in queue reply_57893177120949e6a93cb88e15cd42b4 [ 689.515955] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bdb9282c80c94f1dab41ba84cef337be [ 689.549842] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 30866033047948b59577ac1982b24a87 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 689.586478] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30866033047948b59577ac1982b24a87 [ 689.738472] env[64020]: DEBUG oslo_concurrency.lockutils [req-f6bf8748-f64a-4bd3-bb75-f4d0d3f72c0c req-d22b452b-4942-47b5-9e01-2b2204979447 service nova] Releasing lock "refresh_cache-d4d0d50d-8479-47f7-ba29-114ef10764cc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 689.739009] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-d4d0d50d-8479-47f7-ba29-114ef10764cc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 689.739234] env[64020]: DEBUG nova.network.neutron [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 689.739743] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 165a1da400e24998945b25e228b3006a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 689.746821] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 165a1da400e24998945b25e228b3006a [ 690.010379] env[64020]: DEBUG nova.scheduler.client.report [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 690.012882] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 9cc3c6ab0f6a4273ab9b719d0fcb79c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 690.027029] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9cc3c6ab0f6a4273ab9b719d0fcb79c1 [ 690.053151] env[64020]: DEBUG nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 690.078421] env[64020]: DEBUG nova.virt.hardware [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 690.078672] env[64020]: DEBUG nova.virt.hardware [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 690.079742] env[64020]: DEBUG nova.virt.hardware [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 690.079742] env[64020]: DEBUG nova.virt.hardware [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 690.079742] env[64020]: DEBUG nova.virt.hardware [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 690.079742] env[64020]: DEBUG nova.virt.hardware [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 690.079742] env[64020]: DEBUG nova.virt.hardware [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 690.080016] env[64020]: DEBUG nova.virt.hardware [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 690.080016] env[64020]: DEBUG nova.virt.hardware [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 690.080016] env[64020]: DEBUG nova.virt.hardware [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 690.080384] env[64020]: DEBUG nova.virt.hardware [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 690.081347] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66b6b679-cafa-4527-9a45-5254a3570394 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.090018] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83d00917-2ada-4001-8616-26ba26df81e2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.264991] env[64020]: DEBUG nova.network.neutron [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 690.352337] env[64020]: DEBUG nova.compute.manager [req-91124220-0ae1-4919-8523-60d37f4f02e9 req-f55a12fe-d167-4319-9e68-e20641afabba service nova] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Received event network-changed-7d184f73-764c-4ec8-84ce-066b295da5fb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 690.352612] env[64020]: DEBUG nova.compute.manager [req-91124220-0ae1-4919-8523-60d37f4f02e9 req-f55a12fe-d167-4319-9e68-e20641afabba service nova] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Refreshing instance network info cache due to event network-changed-7d184f73-764c-4ec8-84ce-066b295da5fb. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 690.352886] env[64020]: DEBUG oslo_concurrency.lockutils [req-91124220-0ae1-4919-8523-60d37f4f02e9 req-f55a12fe-d167-4319-9e68-e20641afabba service nova] Acquiring lock "refresh_cache-7437f3e0-0112-4640-9076-5f5f864ec230" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 690.353091] env[64020]: DEBUG oslo_concurrency.lockutils [req-91124220-0ae1-4919-8523-60d37f4f02e9 req-f55a12fe-d167-4319-9e68-e20641afabba service nova] Acquired lock "refresh_cache-7437f3e0-0112-4640-9076-5f5f864ec230" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 690.353675] env[64020]: DEBUG nova.network.neutron [req-91124220-0ae1-4919-8523-60d37f4f02e9 req-f55a12fe-d167-4319-9e68-e20641afabba service nova] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Refreshing network info cache for port 7d184f73-764c-4ec8-84ce-066b295da5fb {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 690.354395] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-91124220-0ae1-4919-8523-60d37f4f02e9 req-f55a12fe-d167-4319-9e68-e20641afabba service nova] Expecting reply to msg ac6a1805c6014126b26195123b88120c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 690.361107] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ac6a1805c6014126b26195123b88120c [ 690.448354] env[64020]: DEBUG nova.network.neutron [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 690.449191] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 405328ae47fa4f469adfbf229e4f2ec0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 690.457336] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 405328ae47fa4f469adfbf229e4f2ec0 [ 690.515282] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.482s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 690.516022] env[64020]: DEBUG nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 690.518073] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg f66cd90f8682495d94a08c489c36a852 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 690.519985] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.546s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 690.521493] env[64020]: INFO nova.compute.claims [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 690.523147] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg b65b6befcf854ce1bb76bb4c48780fcd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 690.593698] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f66cd90f8682495d94a08c489c36a852 [ 690.594279] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b65b6befcf854ce1bb76bb4c48780fcd [ 690.734977] env[64020]: ERROR nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 7d184f73-764c-4ec8-84ce-066b295da5fb, please check neutron logs for more information. [ 690.734977] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 690.734977] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 690.734977] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 690.734977] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 690.734977] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 690.734977] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 690.734977] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 690.734977] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 690.734977] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 690.734977] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 690.734977] env[64020]: ERROR nova.compute.manager raise self.value [ 690.734977] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 690.734977] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 690.734977] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 690.734977] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 690.735475] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 690.735475] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 690.735475] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 7d184f73-764c-4ec8-84ce-066b295da5fb, please check neutron logs for more information. [ 690.735475] env[64020]: ERROR nova.compute.manager [ 690.735475] env[64020]: Traceback (most recent call last): [ 690.735475] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 690.735475] env[64020]: listener.cb(fileno) [ 690.735475] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 690.735475] env[64020]: result = function(*args, **kwargs) [ 690.735475] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 690.735475] env[64020]: return func(*args, **kwargs) [ 690.735475] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 690.735475] env[64020]: raise e [ 690.735475] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 690.735475] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 690.735475] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 690.735475] env[64020]: created_port_ids = self._update_ports_for_instance( [ 690.735475] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 690.735475] env[64020]: with excutils.save_and_reraise_exception(): [ 690.735475] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 690.735475] env[64020]: self.force_reraise() [ 690.735475] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 690.735475] env[64020]: raise self.value [ 690.735475] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 690.735475] env[64020]: updated_port = self._update_port( [ 690.735475] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 690.735475] env[64020]: _ensure_no_port_binding_failure(port) [ 690.735475] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 690.735475] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 690.736268] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 7d184f73-764c-4ec8-84ce-066b295da5fb, please check neutron logs for more information. [ 690.736268] env[64020]: Removing descriptor: 16 [ 690.736268] env[64020]: ERROR nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 7d184f73-764c-4ec8-84ce-066b295da5fb, please check neutron logs for more information. [ 690.736268] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Traceback (most recent call last): [ 690.736268] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 690.736268] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] yield resources [ 690.736268] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 690.736268] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] self.driver.spawn(context, instance, image_meta, [ 690.736268] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 690.736268] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] self._vmops.spawn(context, instance, image_meta, injected_files, [ 690.736268] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 690.736268] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] vm_ref = self.build_virtual_machine(instance, [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] vif_infos = vmwarevif.get_vif_info(self._session, [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] for vif in network_info: [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] return self._sync_wrapper(fn, *args, **kwargs) [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] self.wait() [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] self[:] = self._gt.wait() [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] return self._exit_event.wait() [ 690.736661] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] result = hub.switch() [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] return self.greenlet.switch() [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] result = function(*args, **kwargs) [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] return func(*args, **kwargs) [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] raise e [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] nwinfo = self.network_api.allocate_for_instance( [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 690.737099] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] created_port_ids = self._update_ports_for_instance( [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] with excutils.save_and_reraise_exception(): [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] self.force_reraise() [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] raise self.value [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] updated_port = self._update_port( [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] _ensure_no_port_binding_failure(port) [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 690.737448] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] raise exception.PortBindingFailed(port_id=port['id']) [ 690.737823] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] nova.exception.PortBindingFailed: Binding failed for port 7d184f73-764c-4ec8-84ce-066b295da5fb, please check neutron logs for more information. [ 690.737823] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] [ 690.737823] env[64020]: INFO nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Terminating instance [ 690.738779] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquiring lock "refresh_cache-7437f3e0-0112-4640-9076-5f5f864ec230" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 690.755837] env[64020]: DEBUG nova.compute.manager [req-184dfaa5-1f05-4416-9896-60ec713154e1 req-4222407e-3c49-4f98-9cff-bc47056dd433 service nova] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Received event network-vif-deleted-b7cef9b2-89e8-4bfc-8a6b-fe379e46900f {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 690.885351] env[64020]: DEBUG nova.network.neutron [req-91124220-0ae1-4919-8523-60d37f4f02e9 req-f55a12fe-d167-4319-9e68-e20641afabba service nova] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 690.951516] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-d4d0d50d-8479-47f7-ba29-114ef10764cc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 690.951924] env[64020]: DEBUG nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 690.952229] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 690.952506] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b40167f9-300c-457b-9fc2-18415a6dfd88 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.963614] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3959fa6-376f-45be-83f6-659d4905d150 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.987877] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d4d0d50d-8479-47f7-ba29-114ef10764cc could not be found. [ 690.988065] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 690.988490] env[64020]: INFO nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Took 0.04 seconds to destroy the instance on the hypervisor. [ 690.988586] env[64020]: DEBUG oslo.service.loopingcall [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 690.988800] env[64020]: DEBUG nova.compute.manager [-] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 690.988905] env[64020]: DEBUG nova.network.neutron [-] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 691.010536] env[64020]: DEBUG nova.network.neutron [-] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 691.011033] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 2f69dd7039ca4c2aa47024662477ff69 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 691.018032] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f69dd7039ca4c2aa47024662477ff69 [ 691.026009] env[64020]: DEBUG nova.compute.utils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 691.026605] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg a5c27c7008ab49b6b53c9e8499e6a18e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 691.027414] env[64020]: DEBUG nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 691.027581] env[64020]: DEBUG nova.network.neutron [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 691.030927] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 16fbe8e1439f4528ba93ec2838a7b80a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 691.037232] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a5c27c7008ab49b6b53c9e8499e6a18e [ 691.037686] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16fbe8e1439f4528ba93ec2838a7b80a [ 691.070758] env[64020]: DEBUG nova.network.neutron [req-91124220-0ae1-4919-8523-60d37f4f02e9 req-f55a12fe-d167-4319-9e68-e20641afabba service nova] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 691.071241] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-91124220-0ae1-4919-8523-60d37f4f02e9 req-f55a12fe-d167-4319-9e68-e20641afabba service nova] Expecting reply to msg c14003d6128644c597c27e811e25d870 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 691.079799] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c14003d6128644c597c27e811e25d870 [ 691.106620] env[64020]: DEBUG nova.policy [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd907d44262b84c30a160a4acbdb8634e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8ef724836475487fb78dee9eebd2572e', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 691.513366] env[64020]: DEBUG nova.network.neutron [-] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 691.513638] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 2108eb7567ab4b08ae0c3848f800ec34 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 691.522009] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2108eb7567ab4b08ae0c3848f800ec34 [ 691.530427] env[64020]: DEBUG nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 691.532131] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg d0246d661f274a9987d1e0a474166f1c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 691.569128] env[64020]: DEBUG nova.network.neutron [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Successfully created port: 9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 691.572962] env[64020]: DEBUG oslo_concurrency.lockutils [req-91124220-0ae1-4919-8523-60d37f4f02e9 req-f55a12fe-d167-4319-9e68-e20641afabba service nova] Releasing lock "refresh_cache-7437f3e0-0112-4640-9076-5f5f864ec230" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 691.573342] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquired lock "refresh_cache-7437f3e0-0112-4640-9076-5f5f864ec230" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 691.573520] env[64020]: DEBUG nova.network.neutron [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 691.573972] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 40f1553cdd6845a1ae791c4e67ed3150 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 691.575012] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0246d661f274a9987d1e0a474166f1c [ 691.587536] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40f1553cdd6845a1ae791c4e67ed3150 [ 692.016111] env[64020]: INFO nova.compute.manager [-] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Took 1.03 seconds to deallocate network for instance. [ 692.019116] env[64020]: DEBUG nova.compute.claims [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 692.019293] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 692.024890] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89ff9593-1fd2-40ea-8c88-f476f1a25d86 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.035070] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49b91c31-b179-4dd0-a778-a06efac0960d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.042240] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 9609f2475e634b52ac995f4a07c88a97 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 692.087728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9609f2475e634b52ac995f4a07c88a97 [ 692.089003] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ef2437c-8969-4398-bcc8-ceb8d46363fa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.103240] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-954a79cb-5555-43f2-bbdd-2ad42793c119 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.123411] env[64020]: DEBUG nova.compute.provider_tree [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 692.123967] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 7f7b12c988744090a7156f89449d4d04 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 692.128260] env[64020]: DEBUG nova.network.neutron [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 692.133461] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f7b12c988744090a7156f89449d4d04 [ 692.289241] env[64020]: DEBUG nova.network.neutron [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 692.289241] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 153ebd664c664bf283e74a8beed9de8c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 692.299313] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 153ebd664c664bf283e74a8beed9de8c [ 692.386336] env[64020]: DEBUG nova.compute.manager [req-7009bd78-2c4f-4041-af82-027f6618fec6 req-df6c9423-d2c4-4b64-99c3-3e74895c27df service nova] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Received event network-vif-deleted-7d184f73-764c-4ec8-84ce-066b295da5fb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 692.558534] env[64020]: DEBUG nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 692.586608] env[64020]: DEBUG nova.virt.hardware [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 692.586859] env[64020]: DEBUG nova.virt.hardware [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 692.587009] env[64020]: DEBUG nova.virt.hardware [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 692.587187] env[64020]: DEBUG nova.virt.hardware [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 692.587326] env[64020]: DEBUG nova.virt.hardware [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 692.587465] env[64020]: DEBUG nova.virt.hardware [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 692.587663] env[64020]: DEBUG nova.virt.hardware [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 692.587811] env[64020]: DEBUG nova.virt.hardware [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 692.587968] env[64020]: DEBUG nova.virt.hardware [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 692.588138] env[64020]: DEBUG nova.virt.hardware [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 692.588303] env[64020]: DEBUG nova.virt.hardware [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 692.589150] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7443c342-13f9-4d63-b65d-b2dadc9dd632 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.600338] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89993c97-54f1-40ac-9ab0-9e67fda63de8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.626563] env[64020]: DEBUG nova.scheduler.client.report [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 692.630072] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 476179b0b8244e34aba2d7c51f4b50de in queue reply_57893177120949e6a93cb88e15cd42b4 [ 692.649024] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 476179b0b8244e34aba2d7c51f4b50de [ 692.792658] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Releasing lock "refresh_cache-7437f3e0-0112-4640-9076-5f5f864ec230" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 692.792993] env[64020]: DEBUG nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 692.793187] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 692.793842] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9591fd28-9634-4247-99ba-a3578bfa9acd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.802491] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99e8b3f6-c61e-472f-9d7d-c1689e9fd0e9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.825820] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7437f3e0-0112-4640-9076-5f5f864ec230 could not be found. [ 692.826046] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 692.826225] env[64020]: INFO nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Took 0.03 seconds to destroy the instance on the hypervisor. [ 692.826657] env[64020]: DEBUG oslo.service.loopingcall [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 692.826800] env[64020]: DEBUG nova.compute.manager [-] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 692.826893] env[64020]: DEBUG nova.network.neutron [-] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 692.849396] env[64020]: DEBUG nova.network.neutron [-] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 692.850008] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5791e9633f444c32a672279213066296 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 692.860576] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5791e9633f444c32a672279213066296 [ 692.919266] env[64020]: DEBUG nova.compute.manager [req-e61ea29b-8d7b-44e5-b5d8-ac7a2ffa38f0 req-818f1e05-3377-47c0-8554-58dc37106918 service nova] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Received event network-changed-9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 692.919486] env[64020]: DEBUG nova.compute.manager [req-e61ea29b-8d7b-44e5-b5d8-ac7a2ffa38f0 req-818f1e05-3377-47c0-8554-58dc37106918 service nova] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Refreshing instance network info cache due to event network-changed-9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 692.921440] env[64020]: DEBUG oslo_concurrency.lockutils [req-e61ea29b-8d7b-44e5-b5d8-ac7a2ffa38f0 req-818f1e05-3377-47c0-8554-58dc37106918 service nova] Acquiring lock "refresh_cache-af8b2b66-c2d6-4bb6-a266-52cb82bac3be" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 692.921631] env[64020]: DEBUG oslo_concurrency.lockutils [req-e61ea29b-8d7b-44e5-b5d8-ac7a2ffa38f0 req-818f1e05-3377-47c0-8554-58dc37106918 service nova] Acquired lock "refresh_cache-af8b2b66-c2d6-4bb6-a266-52cb82bac3be" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 692.921818] env[64020]: DEBUG nova.network.neutron [req-e61ea29b-8d7b-44e5-b5d8-ac7a2ffa38f0 req-818f1e05-3377-47c0-8554-58dc37106918 service nova] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Refreshing network info cache for port 9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 692.922347] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e61ea29b-8d7b-44e5-b5d8-ac7a2ffa38f0 req-818f1e05-3377-47c0-8554-58dc37106918 service nova] Expecting reply to msg 5e76062bfb0f4d19b85d726bddfb1d34 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 692.930870] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e76062bfb0f4d19b85d726bddfb1d34 [ 693.107753] env[64020]: ERROR nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9, please check neutron logs for more information. [ 693.107753] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 693.107753] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 693.107753] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 693.107753] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 693.107753] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 693.107753] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 693.107753] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 693.107753] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 693.107753] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 693.107753] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 693.107753] env[64020]: ERROR nova.compute.manager raise self.value [ 693.107753] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 693.107753] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 693.107753] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 693.107753] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 693.108367] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 693.108367] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 693.108367] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9, please check neutron logs for more information. [ 693.108367] env[64020]: ERROR nova.compute.manager [ 693.108367] env[64020]: Traceback (most recent call last): [ 693.108367] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 693.108367] env[64020]: listener.cb(fileno) [ 693.108367] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 693.108367] env[64020]: result = function(*args, **kwargs) [ 693.108367] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 693.108367] env[64020]: return func(*args, **kwargs) [ 693.108367] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 693.108367] env[64020]: raise e [ 693.108367] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 693.108367] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 693.108367] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 693.108367] env[64020]: created_port_ids = self._update_ports_for_instance( [ 693.108367] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 693.108367] env[64020]: with excutils.save_and_reraise_exception(): [ 693.108367] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 693.108367] env[64020]: self.force_reraise() [ 693.108367] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 693.108367] env[64020]: raise self.value [ 693.108367] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 693.108367] env[64020]: updated_port = self._update_port( [ 693.108367] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 693.108367] env[64020]: _ensure_no_port_binding_failure(port) [ 693.108367] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 693.108367] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 693.109071] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9, please check neutron logs for more information. [ 693.109071] env[64020]: Removing descriptor: 18 [ 693.109071] env[64020]: ERROR nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9, please check neutron logs for more information. [ 693.109071] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Traceback (most recent call last): [ 693.109071] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 693.109071] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] yield resources [ 693.109071] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 693.109071] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] self.driver.spawn(context, instance, image_meta, [ 693.109071] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 693.109071] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] self._vmops.spawn(context, instance, image_meta, injected_files, [ 693.109071] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 693.109071] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] vm_ref = self.build_virtual_machine(instance, [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] vif_infos = vmwarevif.get_vif_info(self._session, [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] for vif in network_info: [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] return self._sync_wrapper(fn, *args, **kwargs) [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] self.wait() [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] self[:] = self._gt.wait() [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] return self._exit_event.wait() [ 693.109361] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] result = hub.switch() [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] return self.greenlet.switch() [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] result = function(*args, **kwargs) [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] return func(*args, **kwargs) [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] raise e [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] nwinfo = self.network_api.allocate_for_instance( [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 693.109661] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] created_port_ids = self._update_ports_for_instance( [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] with excutils.save_and_reraise_exception(): [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] self.force_reraise() [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] raise self.value [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] updated_port = self._update_port( [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] _ensure_no_port_binding_failure(port) [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 693.110015] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] raise exception.PortBindingFailed(port_id=port['id']) [ 693.110335] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] nova.exception.PortBindingFailed: Binding failed for port 9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9, please check neutron logs for more information. [ 693.110335] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] [ 693.110335] env[64020]: INFO nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Terminating instance [ 693.111263] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "refresh_cache-af8b2b66-c2d6-4bb6-a266-52cb82bac3be" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 693.133243] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.614s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 693.133773] env[64020]: DEBUG nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 693.135514] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 0273b120301b445db1bb401d1c731859 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 693.136653] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.728s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 693.152361] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg dba319022e414918aea3213ad9a3b78a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 693.175702] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0273b120301b445db1bb401d1c731859 [ 693.205721] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dba319022e414918aea3213ad9a3b78a [ 693.242420] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquiring lock "995288b5-d79a-4af3-a1e8-3571fff2d356" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 693.242633] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Lock "995288b5-d79a-4af3-a1e8-3571fff2d356" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 693.355578] env[64020]: DEBUG nova.network.neutron [-] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 693.355578] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8437cb0945334ccca7f117c9ada38328 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 693.364575] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8437cb0945334ccca7f117c9ada38328 [ 693.439821] env[64020]: DEBUG nova.network.neutron [req-e61ea29b-8d7b-44e5-b5d8-ac7a2ffa38f0 req-818f1e05-3377-47c0-8554-58dc37106918 service nova] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 693.518860] env[64020]: DEBUG nova.network.neutron [req-e61ea29b-8d7b-44e5-b5d8-ac7a2ffa38f0 req-818f1e05-3377-47c0-8554-58dc37106918 service nova] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 693.519457] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e61ea29b-8d7b-44e5-b5d8-ac7a2ffa38f0 req-818f1e05-3377-47c0-8554-58dc37106918 service nova] Expecting reply to msg 2c49eeca2d3e4716916168c648b67161 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 693.528027] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c49eeca2d3e4716916168c648b67161 [ 693.656067] env[64020]: DEBUG nova.compute.utils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 693.656726] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg d8e9e8eef9764a6d868295d5251bc924 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 693.661836] env[64020]: DEBUG nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 693.662154] env[64020]: DEBUG nova.network.neutron [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 693.665907] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d8e9e8eef9764a6d868295d5251bc924 [ 693.712420] env[64020]: DEBUG nova.policy [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '0cca52a2c83c4404bd4ebbd132a7729b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd2903f8aa642452982780317453a4c49', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 693.856779] env[64020]: INFO nova.compute.manager [-] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Took 1.03 seconds to deallocate network for instance. [ 693.859280] env[64020]: DEBUG nova.compute.claims [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 693.859475] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 694.024241] env[64020]: DEBUG oslo_concurrency.lockutils [req-e61ea29b-8d7b-44e5-b5d8-ac7a2ffa38f0 req-818f1e05-3377-47c0-8554-58dc37106918 service nova] Releasing lock "refresh_cache-af8b2b66-c2d6-4bb6-a266-52cb82bac3be" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 694.024728] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquired lock "refresh_cache-af8b2b66-c2d6-4bb6-a266-52cb82bac3be" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 694.024913] env[64020]: DEBUG nova.network.neutron [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 694.025340] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 0af33935e2304a088d84864625e66ef4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 694.031917] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0af33935e2304a088d84864625e66ef4 [ 694.061458] env[64020]: DEBUG nova.network.neutron [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Successfully created port: 4381921e-bc9e-4540-a293-d742eae29ad3 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 694.158302] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ae08f4e-819f-4a4c-8061-bbccc79c2503 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.162778] env[64020]: DEBUG nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 694.164709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 1ac9c00c9c3b4a74a81e2ce2575bc298 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 694.170491] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24c252ea-7237-4c1a-af35-dbe83e5be76d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.201724] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-045dfcda-657e-4769-a722-3718cccd0451 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.209338] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6785071c-51fa-4d33-871b-629eaad9e77a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.223978] env[64020]: DEBUG nova.compute.provider_tree [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 694.224597] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 577986e72e984d66b72985bb44b871af in queue reply_57893177120949e6a93cb88e15cd42b4 [ 694.235220] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 577986e72e984d66b72985bb44b871af [ 694.242882] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ac9c00c9c3b4a74a81e2ce2575bc298 [ 694.256217] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Acquiring lock "45167852-b7c0-4614-89f1-f8f7fc2078f2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 694.256765] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Lock "45167852-b7c0-4614-89f1-f8f7fc2078f2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 694.543436] env[64020]: DEBUG nova.network.neutron [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 694.622969] env[64020]: DEBUG nova.network.neutron [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 694.623464] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg b1d8c9c5d2cd47da9acea08e8db9f721 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 694.631522] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1d8c9c5d2cd47da9acea08e8db9f721 [ 694.675256] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 4c1081b6092644e1bcc2ec9db6dd2dcc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 694.710309] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c1081b6092644e1bcc2ec9db6dd2dcc [ 694.726908] env[64020]: DEBUG nova.scheduler.client.report [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 694.729551] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg f791bd0d74984c179d44fe8f11931653 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 694.741886] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f791bd0d74984c179d44fe8f11931653 [ 694.979954] env[64020]: ERROR nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4381921e-bc9e-4540-a293-d742eae29ad3, please check neutron logs for more information. [ 694.979954] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 694.979954] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 694.979954] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 694.979954] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 694.979954] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 694.979954] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 694.979954] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 694.979954] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 694.979954] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 694.979954] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 694.979954] env[64020]: ERROR nova.compute.manager raise self.value [ 694.979954] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 694.979954] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 694.979954] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 694.979954] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 694.980406] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 694.980406] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 694.980406] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4381921e-bc9e-4540-a293-d742eae29ad3, please check neutron logs for more information. [ 694.980406] env[64020]: ERROR nova.compute.manager [ 694.980406] env[64020]: Traceback (most recent call last): [ 694.980406] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 694.980406] env[64020]: listener.cb(fileno) [ 694.980406] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 694.980406] env[64020]: result = function(*args, **kwargs) [ 694.980406] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 694.980406] env[64020]: return func(*args, **kwargs) [ 694.980406] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 694.980406] env[64020]: raise e [ 694.980406] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 694.980406] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 694.980406] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 694.980406] env[64020]: created_port_ids = self._update_ports_for_instance( [ 694.980406] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 694.980406] env[64020]: with excutils.save_and_reraise_exception(): [ 694.980406] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 694.980406] env[64020]: self.force_reraise() [ 694.980406] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 694.980406] env[64020]: raise self.value [ 694.980406] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 694.980406] env[64020]: updated_port = self._update_port( [ 694.980406] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 694.980406] env[64020]: _ensure_no_port_binding_failure(port) [ 694.980406] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 694.980406] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 694.981271] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 4381921e-bc9e-4540-a293-d742eae29ad3, please check neutron logs for more information. [ 694.981271] env[64020]: Removing descriptor: 18 [ 695.001404] env[64020]: DEBUG nova.compute.manager [req-011ac405-fde7-4f18-a1fb-3d50eb4c6d42 req-f9d40f17-3264-4a49-858b-cb7d7318c36f service nova] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Received event network-vif-deleted-9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 695.001600] env[64020]: DEBUG nova.compute.manager [req-011ac405-fde7-4f18-a1fb-3d50eb4c6d42 req-f9d40f17-3264-4a49-858b-cb7d7318c36f service nova] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Received event network-changed-4381921e-bc9e-4540-a293-d742eae29ad3 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 695.001753] env[64020]: DEBUG nova.compute.manager [req-011ac405-fde7-4f18-a1fb-3d50eb4c6d42 req-f9d40f17-3264-4a49-858b-cb7d7318c36f service nova] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Refreshing instance network info cache due to event network-changed-4381921e-bc9e-4540-a293-d742eae29ad3. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 695.001998] env[64020]: DEBUG oslo_concurrency.lockutils [req-011ac405-fde7-4f18-a1fb-3d50eb4c6d42 req-f9d40f17-3264-4a49-858b-cb7d7318c36f service nova] Acquiring lock "refresh_cache-583b323c-a4e1-4431-9bc0-d9f4b7354cb6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 695.002091] env[64020]: DEBUG oslo_concurrency.lockutils [req-011ac405-fde7-4f18-a1fb-3d50eb4c6d42 req-f9d40f17-3264-4a49-858b-cb7d7318c36f service nova] Acquired lock "refresh_cache-583b323c-a4e1-4431-9bc0-d9f4b7354cb6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 695.002297] env[64020]: DEBUG nova.network.neutron [req-011ac405-fde7-4f18-a1fb-3d50eb4c6d42 req-f9d40f17-3264-4a49-858b-cb7d7318c36f service nova] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Refreshing network info cache for port 4381921e-bc9e-4540-a293-d742eae29ad3 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 695.002713] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-011ac405-fde7-4f18-a1fb-3d50eb4c6d42 req-f9d40f17-3264-4a49-858b-cb7d7318c36f service nova] Expecting reply to msg 7c680a4ea50a45b8b24481c49e537a1d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 695.009269] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c680a4ea50a45b8b24481c49e537a1d [ 695.125643] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Releasing lock "refresh_cache-af8b2b66-c2d6-4bb6-a266-52cb82bac3be" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 695.126080] env[64020]: DEBUG nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 695.126271] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 695.126556] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5f0aa5b5-fdaa-41c2-b9cd-69217f05f3bd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 695.135415] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dff19c3-229d-41a2-b4ff-7f2f86ff6dba {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 695.157255] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance af8b2b66-c2d6-4bb6-a266-52cb82bac3be could not be found. [ 695.157455] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 695.157701] env[64020]: INFO nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Took 0.03 seconds to destroy the instance on the hypervisor. [ 695.157999] env[64020]: DEBUG oslo.service.loopingcall [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 695.158217] env[64020]: DEBUG nova.compute.manager [-] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 695.158312] env[64020]: DEBUG nova.network.neutron [-] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 695.171817] env[64020]: DEBUG nova.network.neutron [-] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 695.172351] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 63da93f192b44559abd748461d97084b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 695.178513] env[64020]: DEBUG nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 695.180670] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 63da93f192b44559abd748461d97084b [ 695.204069] env[64020]: DEBUG nova.virt.hardware [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 695.204394] env[64020]: DEBUG nova.virt.hardware [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 695.204611] env[64020]: DEBUG nova.virt.hardware [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 695.204854] env[64020]: DEBUG nova.virt.hardware [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 695.205056] env[64020]: DEBUG nova.virt.hardware [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 695.205279] env[64020]: DEBUG nova.virt.hardware [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 695.205550] env[64020]: DEBUG nova.virt.hardware [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 695.205768] env[64020]: DEBUG nova.virt.hardware [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 695.205994] env[64020]: DEBUG nova.virt.hardware [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 695.206213] env[64020]: DEBUG nova.virt.hardware [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 695.206445] env[64020]: DEBUG nova.virt.hardware [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 695.207305] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5dfdc6c7-a8e2-4eaf-b2a3-688893c9f97f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 695.215155] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-445cc1bd-4d10-4456-b263-8100d370bd74 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 695.228906] env[64020]: ERROR nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4381921e-bc9e-4540-a293-d742eae29ad3, please check neutron logs for more information. [ 695.228906] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Traceback (most recent call last): [ 695.228906] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 695.228906] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] yield resources [ 695.228906] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 695.228906] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] self.driver.spawn(context, instance, image_meta, [ 695.228906] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 695.228906] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 695.228906] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 695.228906] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] vm_ref = self.build_virtual_machine(instance, [ 695.228906] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] vif_infos = vmwarevif.get_vif_info(self._session, [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] for vif in network_info: [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] return self._sync_wrapper(fn, *args, **kwargs) [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] self.wait() [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] self[:] = self._gt.wait() [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] return self._exit_event.wait() [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 695.229277] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] current.throw(*self._exc) [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] result = function(*args, **kwargs) [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] return func(*args, **kwargs) [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] raise e [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] nwinfo = self.network_api.allocate_for_instance( [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] created_port_ids = self._update_ports_for_instance( [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] with excutils.save_and_reraise_exception(): [ 695.229669] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 695.230217] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] self.force_reraise() [ 695.230217] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 695.230217] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] raise self.value [ 695.230217] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 695.230217] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] updated_port = self._update_port( [ 695.230217] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 695.230217] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] _ensure_no_port_binding_failure(port) [ 695.230217] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 695.230217] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] raise exception.PortBindingFailed(port_id=port['id']) [ 695.230217] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] nova.exception.PortBindingFailed: Binding failed for port 4381921e-bc9e-4540-a293-d742eae29ad3, please check neutron logs for more information. [ 695.230217] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] [ 695.230217] env[64020]: INFO nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Terminating instance [ 695.231930] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.095s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 695.232537] env[64020]: ERROR nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f5295483-d7ae-46c3-bef5-4edfb2f1ec32, please check neutron logs for more information. [ 695.232537] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Traceback (most recent call last): [ 695.232537] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 695.232537] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] self.driver.spawn(context, instance, image_meta, [ 695.232537] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 695.232537] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 695.232537] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 695.232537] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] vm_ref = self.build_virtual_machine(instance, [ 695.232537] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 695.232537] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] vif_infos = vmwarevif.get_vif_info(self._session, [ 695.232537] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] for vif in network_info: [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] return self._sync_wrapper(fn, *args, **kwargs) [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] self.wait() [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] self[:] = self._gt.wait() [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] return self._exit_event.wait() [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] result = hub.switch() [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 695.232854] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] return self.greenlet.switch() [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] result = function(*args, **kwargs) [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] return func(*args, **kwargs) [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] raise e [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] nwinfo = self.network_api.allocate_for_instance( [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] created_port_ids = self._update_ports_for_instance( [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] with excutils.save_and_reraise_exception(): [ 695.233178] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 695.233511] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] self.force_reraise() [ 695.233511] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 695.233511] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] raise self.value [ 695.233511] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 695.233511] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] updated_port = self._update_port( [ 695.233511] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 695.233511] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] _ensure_no_port_binding_failure(port) [ 695.233511] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 695.233511] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] raise exception.PortBindingFailed(port_id=port['id']) [ 695.233511] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] nova.exception.PortBindingFailed: Binding failed for port f5295483-d7ae-46c3-bef5-4edfb2f1ec32, please check neutron logs for more information. [ 695.233511] env[64020]: ERROR nova.compute.manager [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] [ 695.233967] env[64020]: DEBUG nova.compute.utils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Binding failed for port f5295483-d7ae-46c3-bef5-4edfb2f1ec32, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 695.235414] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Acquiring lock "refresh_cache-583b323c-a4e1-4431-9bc0-d9f4b7354cb6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 695.235874] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.939s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 695.237373] env[64020]: INFO nova.compute.claims [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 695.238913] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 403ff47a1438433ea0c9f0de55b4ac6a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 695.255811] env[64020]: DEBUG nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Build of instance cea1a347-7f1a-4090-b3ad-54ccfbdfea2a was re-scheduled: Binding failed for port f5295483-d7ae-46c3-bef5-4edfb2f1ec32, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 695.256275] env[64020]: DEBUG nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 695.256479] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Acquiring lock "refresh_cache-cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 695.256620] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Acquired lock "refresh_cache-cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 695.256770] env[64020]: DEBUG nova.network.neutron [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 695.257144] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg f4b9a445048348fc9e28703850f2c737 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 695.263033] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f4b9a445048348fc9e28703850f2c737 [ 695.277646] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 403ff47a1438433ea0c9f0de55b4ac6a [ 695.518666] env[64020]: DEBUG nova.network.neutron [req-011ac405-fde7-4f18-a1fb-3d50eb4c6d42 req-f9d40f17-3264-4a49-858b-cb7d7318c36f service nova] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 695.591587] env[64020]: DEBUG nova.network.neutron [req-011ac405-fde7-4f18-a1fb-3d50eb4c6d42 req-f9d40f17-3264-4a49-858b-cb7d7318c36f service nova] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 695.592120] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-011ac405-fde7-4f18-a1fb-3d50eb4c6d42 req-f9d40f17-3264-4a49-858b-cb7d7318c36f service nova] Expecting reply to msg 2547e45709d4491e8f2f3901397599c4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 695.601955] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2547e45709d4491e8f2f3901397599c4 [ 695.674664] env[64020]: DEBUG nova.network.neutron [-] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 695.674990] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 09087e4dcd3849f7bc8b470a069a0ec0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 695.689561] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 09087e4dcd3849f7bc8b470a069a0ec0 [ 695.742689] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 62d96ca3ecad43d1b7a64620eeba6912 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 695.749978] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62d96ca3ecad43d1b7a64620eeba6912 [ 695.773356] env[64020]: DEBUG nova.network.neutron [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 695.852536] env[64020]: DEBUG nova.network.neutron [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 695.853053] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg d2072c7ecda84c669c9bd9721bbb9e8f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 695.861396] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d2072c7ecda84c669c9bd9721bbb9e8f [ 696.094812] env[64020]: DEBUG oslo_concurrency.lockutils [req-011ac405-fde7-4f18-a1fb-3d50eb4c6d42 req-f9d40f17-3264-4a49-858b-cb7d7318c36f service nova] Releasing lock "refresh_cache-583b323c-a4e1-4431-9bc0-d9f4b7354cb6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 696.095242] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Acquired lock "refresh_cache-583b323c-a4e1-4431-9bc0-d9f4b7354cb6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 696.095426] env[64020]: DEBUG nova.network.neutron [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 696.095862] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 7d440524b3234a4a97355609c4b45aaa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 696.102812] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d440524b3234a4a97355609c4b45aaa [ 696.176969] env[64020]: INFO nova.compute.manager [-] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Took 1.02 seconds to deallocate network for instance. [ 696.180787] env[64020]: DEBUG nova.compute.claims [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 696.180981] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 696.355245] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Releasing lock "refresh_cache-cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 696.355487] env[64020]: DEBUG nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 696.355664] env[64020]: DEBUG nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 696.355831] env[64020]: DEBUG nova.network.neutron [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 696.370015] env[64020]: DEBUG nova.network.neutron [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 696.370572] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 8920d3c494004d2da1c03e9c5c13a39c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 696.377339] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8920d3c494004d2da1c03e9c5c13a39c [ 696.599916] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7dc64ff-a0a3-4dc6-8d2e-2916f5652881 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.608149] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55337b47-28db-43e7-baa2-16b6614acc17 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.614919] env[64020]: DEBUG nova.network.neutron [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 696.641214] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-289c0b65-24d3-45e0-9b70-5de2599aaa73 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.648583] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2f7f0a8-a7ad-4e05-8281-c9c25e019dcb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.661843] env[64020]: DEBUG nova.compute.provider_tree [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 696.662340] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg c2117493bbac410cb897d4136d66840e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 696.669867] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2117493bbac410cb897d4136d66840e [ 696.710208] env[64020]: DEBUG nova.network.neutron [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 696.710697] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 151b3feddd4d47959d60365fe54b015b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 696.718743] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 151b3feddd4d47959d60365fe54b015b [ 696.875006] env[64020]: DEBUG nova.network.neutron [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 696.875539] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg b8b6fbb0fdc0442ea376e588c2f19c53 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 696.883577] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8b6fbb0fdc0442ea376e588c2f19c53 [ 697.050105] env[64020]: DEBUG nova.compute.manager [req-6c6e3fd7-dc77-41e8-96e6-020b77339c67 req-70e72ff7-71e5-4889-b745-2b4f3d65bd89 service nova] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Received event network-vif-deleted-4381921e-bc9e-4540-a293-d742eae29ad3 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 697.165041] env[64020]: DEBUG nova.scheduler.client.report [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 697.167517] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg f4382b2d6c6b47fdb1a585e8794b00d5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 697.179672] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f4382b2d6c6b47fdb1a585e8794b00d5 [ 697.212904] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Releasing lock "refresh_cache-583b323c-a4e1-4431-9bc0-d9f4b7354cb6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 697.213312] env[64020]: DEBUG nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 697.213509] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 697.214015] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-38cee0b6-0de6-4063-8981-1fe935873a02 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.222524] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ee115ea-751b-4323-a463-5221e83b64e4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.243124] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 583b323c-a4e1-4431-9bc0-d9f4b7354cb6 could not be found. [ 697.243326] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 697.243501] env[64020]: INFO nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Took 0.03 seconds to destroy the instance on the hypervisor. [ 697.243740] env[64020]: DEBUG oslo.service.loopingcall [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 697.243945] env[64020]: DEBUG nova.compute.manager [-] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 697.244048] env[64020]: DEBUG nova.network.neutron [-] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 697.258411] env[64020]: DEBUG nova.network.neutron [-] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 697.258865] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg e4ae48cd32224d1e9cc473eb736f020c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 697.266407] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e4ae48cd32224d1e9cc473eb736f020c [ 697.378071] env[64020]: INFO nova.compute.manager [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] [instance: cea1a347-7f1a-4090-b3ad-54ccfbdfea2a] Took 1.02 seconds to deallocate network for instance. [ 697.380214] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg c77643e70efb40bab82118a80088f5d3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 697.412823] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c77643e70efb40bab82118a80088f5d3 [ 697.672042] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.434s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 697.672042] env[64020]: DEBUG nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 697.672042] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 33fb8af952664714a84ef3d58a9a4f78 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 697.672940] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.864s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 697.674290] env[64020]: INFO nova.compute.claims [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 697.675722] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 33bd9be023b74f98a62643be53e17b4f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 697.701716] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33fb8af952664714a84ef3d58a9a4f78 [ 697.712537] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33bd9be023b74f98a62643be53e17b4f [ 697.760927] env[64020]: DEBUG nova.network.neutron [-] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 697.761404] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a14a1ea5b99d464d9b0c0834fc7b2886 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 697.769414] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a14a1ea5b99d464d9b0c0834fc7b2886 [ 697.885719] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 7a0f82f9cd424a0babb608c8fe03ad9c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 697.919796] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a0f82f9cd424a0babb608c8fe03ad9c [ 698.179473] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg c276dc5242ae42d8910f3513b17495f4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 698.181848] env[64020]: DEBUG nova.compute.utils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 698.182522] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 18ae507a7010422a8738f345bddc08b1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 698.183967] env[64020]: DEBUG nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 698.184217] env[64020]: DEBUG nova.network.neutron [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 698.186999] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c276dc5242ae42d8910f3513b17495f4 [ 698.192471] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18ae507a7010422a8738f345bddc08b1 [ 698.226836] env[64020]: DEBUG nova.policy [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd907d44262b84c30a160a4acbdb8634e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8ef724836475487fb78dee9eebd2572e', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 698.263691] env[64020]: INFO nova.compute.manager [-] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Took 1.02 seconds to deallocate network for instance. [ 698.266135] env[64020]: DEBUG nova.compute.claims [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 698.266318] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 698.405517] env[64020]: INFO nova.scheduler.client.report [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Deleted allocations for instance cea1a347-7f1a-4090-b3ad-54ccfbdfea2a [ 698.431293] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Expecting reply to msg 16f8f6d3396b456d9a41c666c4b7f0aa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 698.441455] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16f8f6d3396b456d9a41c666c4b7f0aa [ 698.509457] env[64020]: DEBUG nova.network.neutron [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Successfully created port: 6d7c92ef-7904-4880-a4d2-89b79f6aaae6 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 698.684906] env[64020]: DEBUG nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 698.686640] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg b677fcdce2914b5ea867c1b4aa433954 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 698.747826] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b677fcdce2914b5ea867c1b4aa433954 [ 698.929145] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3d17dfd-fdb9-46cc-871e-44b511b2ed20 tempest-ServersWithSpecificFlavorTestJSON-423255920 tempest-ServersWithSpecificFlavorTestJSON-423255920-project-member] Lock "cea1a347-7f1a-4090-b3ad-54ccfbdfea2a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 92.375s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 698.929888] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 71953c42bbf24b92af2909a52989e4c6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 698.943600] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 71953c42bbf24b92af2909a52989e4c6 [ 699.170524] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-865d5925-d416-4076-8aca-ab9b95cca7da {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.178630] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1e2c7b2-5e4d-4f9c-b968-9551bcdb842b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.213407] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 9203f485226a42bda99bf54519e47dbf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 699.215680] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd6b791e-7db8-4a20-82c0-08bf1f611e1c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.223275] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4bf1dc25-9d35-4b58-9d83-2f2ddcf83016 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.236593] env[64020]: DEBUG nova.compute.provider_tree [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 699.237275] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg de2b77b2b2f4469ba4b915b1c995744f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 699.243637] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9203f485226a42bda99bf54519e47dbf [ 699.249903] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de2b77b2b2f4469ba4b915b1c995744f [ 699.396221] env[64020]: ERROR nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6d7c92ef-7904-4880-a4d2-89b79f6aaae6, please check neutron logs for more information. [ 699.396221] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 699.396221] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 699.396221] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 699.396221] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 699.396221] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 699.396221] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 699.396221] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 699.396221] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 699.396221] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 699.396221] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 699.396221] env[64020]: ERROR nova.compute.manager raise self.value [ 699.396221] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 699.396221] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 699.396221] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 699.396221] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 699.396700] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 699.396700] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 699.396700] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6d7c92ef-7904-4880-a4d2-89b79f6aaae6, please check neutron logs for more information. [ 699.396700] env[64020]: ERROR nova.compute.manager [ 699.396700] env[64020]: Traceback (most recent call last): [ 699.396700] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 699.396700] env[64020]: listener.cb(fileno) [ 699.396700] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 699.396700] env[64020]: result = function(*args, **kwargs) [ 699.396700] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 699.396700] env[64020]: return func(*args, **kwargs) [ 699.396700] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 699.396700] env[64020]: raise e [ 699.396700] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 699.396700] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 699.396700] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 699.396700] env[64020]: created_port_ids = self._update_ports_for_instance( [ 699.396700] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 699.396700] env[64020]: with excutils.save_and_reraise_exception(): [ 699.396700] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 699.396700] env[64020]: self.force_reraise() [ 699.396700] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 699.396700] env[64020]: raise self.value [ 699.396700] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 699.396700] env[64020]: updated_port = self._update_port( [ 699.396700] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 699.396700] env[64020]: _ensure_no_port_binding_failure(port) [ 699.396700] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 699.396700] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 699.397492] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 6d7c92ef-7904-4880-a4d2-89b79f6aaae6, please check neutron logs for more information. [ 699.397492] env[64020]: Removing descriptor: 18 [ 699.424312] env[64020]: DEBUG nova.compute.manager [req-41102c27-0ef6-4607-8627-4fe3c0ba8bc0 req-2d89a068-2bc3-494a-a59f-3bf952013667 service nova] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Received event network-changed-6d7c92ef-7904-4880-a4d2-89b79f6aaae6 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 699.424541] env[64020]: DEBUG nova.compute.manager [req-41102c27-0ef6-4607-8627-4fe3c0ba8bc0 req-2d89a068-2bc3-494a-a59f-3bf952013667 service nova] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Refreshing instance network info cache due to event network-changed-6d7c92ef-7904-4880-a4d2-89b79f6aaae6. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 699.424817] env[64020]: DEBUG oslo_concurrency.lockutils [req-41102c27-0ef6-4607-8627-4fe3c0ba8bc0 req-2d89a068-2bc3-494a-a59f-3bf952013667 service nova] Acquiring lock "refresh_cache-8368f535-f3d4-4b03-98ef-20b0bf91d25c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 699.425154] env[64020]: DEBUG oslo_concurrency.lockutils [req-41102c27-0ef6-4607-8627-4fe3c0ba8bc0 req-2d89a068-2bc3-494a-a59f-3bf952013667 service nova] Acquired lock "refresh_cache-8368f535-f3d4-4b03-98ef-20b0bf91d25c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 699.425405] env[64020]: DEBUG nova.network.neutron [req-41102c27-0ef6-4607-8627-4fe3c0ba8bc0 req-2d89a068-2bc3-494a-a59f-3bf952013667 service nova] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Refreshing network info cache for port 6d7c92ef-7904-4880-a4d2-89b79f6aaae6 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 699.425858] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-41102c27-0ef6-4607-8627-4fe3c0ba8bc0 req-2d89a068-2bc3-494a-a59f-3bf952013667 service nova] Expecting reply to msg e125351cc5f74f199286a15e0313121f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 699.431648] env[64020]: DEBUG nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 699.433830] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 342ddfb59a294f57b2fa225373526dd6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 699.435076] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e125351cc5f74f199286a15e0313121f [ 699.475873] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 342ddfb59a294f57b2fa225373526dd6 [ 699.716506] env[64020]: DEBUG nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 699.741520] env[64020]: DEBUG nova.scheduler.client.report [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 699.744213] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 92540a57a4d941058a46c95403b10fe0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 699.754488] env[64020]: DEBUG nova.virt.hardware [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 699.755016] env[64020]: DEBUG nova.virt.hardware [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 699.755233] env[64020]: DEBUG nova.virt.hardware [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 699.755470] env[64020]: DEBUG nova.virt.hardware [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 699.755669] env[64020]: DEBUG nova.virt.hardware [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 699.756133] env[64020]: DEBUG nova.virt.hardware [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 699.756424] env[64020]: DEBUG nova.virt.hardware [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 699.756647] env[64020]: DEBUG nova.virt.hardware [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 699.757367] env[64020]: DEBUG nova.virt.hardware [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 699.757609] env[64020]: DEBUG nova.virt.hardware [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 699.757845] env[64020]: DEBUG nova.virt.hardware [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 699.759258] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-394353ae-21de-4b2f-a980-cc4fe87ec28b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.763049] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92540a57a4d941058a46c95403b10fe0 [ 699.769637] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5624f9cd-7116-4315-a99a-8d89a5e91049 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.784917] env[64020]: ERROR nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6d7c92ef-7904-4880-a4d2-89b79f6aaae6, please check neutron logs for more information. [ 699.784917] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Traceback (most recent call last): [ 699.784917] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 699.784917] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] yield resources [ 699.784917] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 699.784917] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] self.driver.spawn(context, instance, image_meta, [ 699.784917] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 699.784917] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 699.784917] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 699.784917] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] vm_ref = self.build_virtual_machine(instance, [ 699.784917] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] vif_infos = vmwarevif.get_vif_info(self._session, [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] for vif in network_info: [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] return self._sync_wrapper(fn, *args, **kwargs) [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] self.wait() [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] self[:] = self._gt.wait() [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] return self._exit_event.wait() [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 699.785275] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] current.throw(*self._exc) [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] result = function(*args, **kwargs) [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] return func(*args, **kwargs) [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] raise e [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] nwinfo = self.network_api.allocate_for_instance( [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] created_port_ids = self._update_ports_for_instance( [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] with excutils.save_and_reraise_exception(): [ 699.785604] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 699.785932] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] self.force_reraise() [ 699.785932] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 699.785932] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] raise self.value [ 699.785932] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 699.785932] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] updated_port = self._update_port( [ 699.785932] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 699.785932] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] _ensure_no_port_binding_failure(port) [ 699.785932] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 699.785932] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] raise exception.PortBindingFailed(port_id=port['id']) [ 699.785932] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] nova.exception.PortBindingFailed: Binding failed for port 6d7c92ef-7904-4880-a4d2-89b79f6aaae6, please check neutron logs for more information. [ 699.785932] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] [ 699.785932] env[64020]: INFO nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Terminating instance [ 699.788157] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "refresh_cache-8368f535-f3d4-4b03-98ef-20b0bf91d25c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 699.944184] env[64020]: DEBUG nova.network.neutron [req-41102c27-0ef6-4607-8627-4fe3c0ba8bc0 req-2d89a068-2bc3-494a-a59f-3bf952013667 service nova] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 699.955150] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 700.030257] env[64020]: DEBUG nova.network.neutron [req-41102c27-0ef6-4607-8627-4fe3c0ba8bc0 req-2d89a068-2bc3-494a-a59f-3bf952013667 service nova] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 700.030910] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-41102c27-0ef6-4607-8627-4fe3c0ba8bc0 req-2d89a068-2bc3-494a-a59f-3bf952013667 service nova] Expecting reply to msg 62ae188bc19947078b277a4a95fcc63c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 700.039551] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62ae188bc19947078b277a4a95fcc63c [ 700.247218] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.574s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 700.247872] env[64020]: DEBUG nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 700.249642] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 57fd4cccc865465a8ebc357f9b92b5f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 700.250744] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.392s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 700.252554] env[64020]: INFO nova.compute.claims [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 700.254442] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg bbb1ff10cd844eea93522fb09a6fff6b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 700.292898] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bbb1ff10cd844eea93522fb09a6fff6b [ 700.297265] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 57fd4cccc865465a8ebc357f9b92b5f5 [ 700.533969] env[64020]: DEBUG oslo_concurrency.lockutils [req-41102c27-0ef6-4607-8627-4fe3c0ba8bc0 req-2d89a068-2bc3-494a-a59f-3bf952013667 service nova] Releasing lock "refresh_cache-8368f535-f3d4-4b03-98ef-20b0bf91d25c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 700.534451] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquired lock "refresh_cache-8368f535-f3d4-4b03-98ef-20b0bf91d25c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 700.534646] env[64020]: DEBUG nova.network.neutron [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 700.535073] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg cd64c872bef34716a56858eb2dba3a40 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 700.543438] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd64c872bef34716a56858eb2dba3a40 [ 700.758443] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 7649f8150f714c7280aa7c32c3ad57f0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 700.760233] env[64020]: DEBUG nova.compute.utils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 700.760813] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 7c2ce3716bb440ee84aa6adbd3bba87c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 700.768377] env[64020]: DEBUG nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 700.768643] env[64020]: DEBUG nova.network.neutron [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 700.776440] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7649f8150f714c7280aa7c32c3ad57f0 [ 700.782739] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c2ce3716bb440ee84aa6adbd3bba87c [ 700.818516] env[64020]: DEBUG nova.policy [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd907d44262b84c30a160a4acbdb8634e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8ef724836475487fb78dee9eebd2572e', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 701.064355] env[64020]: DEBUG nova.network.neutron [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 701.162174] env[64020]: DEBUG nova.network.neutron [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Successfully created port: 89cee47c-37ae-4a0d-8184-0c7c53ff7bb1 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 701.240189] env[64020]: DEBUG nova.network.neutron [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 701.240798] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 9abd38c45a32420183dcdf6f79d53185 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 701.249694] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9abd38c45a32420183dcdf6f79d53185 [ 701.271537] env[64020]: DEBUG nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 701.273420] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 9c063e763de34347ac1dc7db91616b04 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 701.307418] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c063e763de34347ac1dc7db91616b04 [ 701.494852] env[64020]: DEBUG nova.compute.manager [req-d8e4f5d1-2805-4479-8312-8da1601dcb82 req-d9d463f9-8149-43a7-a2b6-0e5411988b53 service nova] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Received event network-vif-deleted-6d7c92ef-7904-4880-a4d2-89b79f6aaae6 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 701.727809] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef949ff1-b2dc-4350-8dfa-44579dfbe2a4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.735538] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f867ac76-63fb-4646-aa23-6c74c259f34c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.769405] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Releasing lock "refresh_cache-8368f535-f3d4-4b03-98ef-20b0bf91d25c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 701.769822] env[64020]: DEBUG nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 701.770018] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 701.770844] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-aaa66f98-6eb8-42d5-9e22-6a4e68fea588 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.773693] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7849aa04-2e3b-478e-963b-3b55144aa5bd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.786175] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg c6b7e9d44e384e43aaf96f2b223d55a5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 701.801213] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb2abcaa-6336-4d45-840c-6ff878fd0b1f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.816335] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e181dda7-e352-4d52-b6d5-5b9304fb0982 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.826066] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c6b7e9d44e384e43aaf96f2b223d55a5 [ 701.826926] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 8368f535-f3d4-4b03-98ef-20b0bf91d25c could not be found. [ 701.827175] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 701.827384] env[64020]: INFO nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Took 0.06 seconds to destroy the instance on the hypervisor. [ 701.827876] env[64020]: DEBUG oslo.service.loopingcall [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 701.828854] env[64020]: DEBUG nova.compute.manager [-] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 701.828995] env[64020]: DEBUG nova.network.neutron [-] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 701.839679] env[64020]: DEBUG nova.compute.provider_tree [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 701.840299] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 3d5586296e13433bb72ed973f971f1bd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 701.849130] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d5586296e13433bb72ed973f971f1bd [ 701.865048] env[64020]: DEBUG nova.network.neutron [-] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 701.865747] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 13a7cbd96c4d4a47ba1b60226e3706ab in queue reply_57893177120949e6a93cb88e15cd42b4 [ 701.872959] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 13a7cbd96c4d4a47ba1b60226e3706ab [ 702.299903] env[64020]: DEBUG nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 702.332195] env[64020]: DEBUG nova.virt.hardware [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:19Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='84',id=12,is_public=True,memory_mb=256,name='m1.micro',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 702.332575] env[64020]: DEBUG nova.virt.hardware [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 702.332738] env[64020]: DEBUG nova.virt.hardware [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 702.332919] env[64020]: DEBUG nova.virt.hardware [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 702.333061] env[64020]: DEBUG nova.virt.hardware [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 702.333200] env[64020]: DEBUG nova.virt.hardware [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 702.333402] env[64020]: DEBUG nova.virt.hardware [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 702.333557] env[64020]: DEBUG nova.virt.hardware [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 702.333717] env[64020]: DEBUG nova.virt.hardware [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 702.333871] env[64020]: DEBUG nova.virt.hardware [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 702.334039] env[64020]: DEBUG nova.virt.hardware [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 702.334903] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c079b937-a239-423b-809d-0a38306f37b9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.342880] env[64020]: DEBUG nova.scheduler.client.report [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 702.345231] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg d307eb4e006e4342ac297550ebe9ec0d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 702.347113] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5750f9db-274f-435f-b97b-e71dd7707dbb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.361541] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d307eb4e006e4342ac297550ebe9ec0d [ 702.367589] env[64020]: DEBUG nova.network.neutron [-] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 702.368026] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3c9ac9d8eca24ed793a81d4047e6c7d2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 702.375388] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c9ac9d8eca24ed793a81d4047e6c7d2 [ 702.744907] env[64020]: ERROR nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 89cee47c-37ae-4a0d-8184-0c7c53ff7bb1, please check neutron logs for more information. [ 702.744907] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 702.744907] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 702.744907] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 702.744907] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 702.744907] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 702.744907] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 702.744907] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 702.744907] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 702.744907] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 702.744907] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 702.744907] env[64020]: ERROR nova.compute.manager raise self.value [ 702.744907] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 702.744907] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 702.744907] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 702.744907] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 702.745370] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 702.745370] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 702.745370] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 89cee47c-37ae-4a0d-8184-0c7c53ff7bb1, please check neutron logs for more information. [ 702.745370] env[64020]: ERROR nova.compute.manager [ 702.745370] env[64020]: Traceback (most recent call last): [ 702.745370] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 702.745370] env[64020]: listener.cb(fileno) [ 702.745370] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 702.745370] env[64020]: result = function(*args, **kwargs) [ 702.745370] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 702.745370] env[64020]: return func(*args, **kwargs) [ 702.745370] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 702.745370] env[64020]: raise e [ 702.745370] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 702.745370] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 702.745370] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 702.745370] env[64020]: created_port_ids = self._update_ports_for_instance( [ 702.745370] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 702.745370] env[64020]: with excutils.save_and_reraise_exception(): [ 702.745370] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 702.745370] env[64020]: self.force_reraise() [ 702.745370] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 702.745370] env[64020]: raise self.value [ 702.745370] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 702.745370] env[64020]: updated_port = self._update_port( [ 702.745370] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 702.745370] env[64020]: _ensure_no_port_binding_failure(port) [ 702.745370] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 702.745370] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 702.746161] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 89cee47c-37ae-4a0d-8184-0c7c53ff7bb1, please check neutron logs for more information. [ 702.746161] env[64020]: Removing descriptor: 18 [ 702.746161] env[64020]: ERROR nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 89cee47c-37ae-4a0d-8184-0c7c53ff7bb1, please check neutron logs for more information. [ 702.746161] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Traceback (most recent call last): [ 702.746161] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 702.746161] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] yield resources [ 702.746161] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 702.746161] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] self.driver.spawn(context, instance, image_meta, [ 702.746161] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 702.746161] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 702.746161] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 702.746161] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] vm_ref = self.build_virtual_machine(instance, [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] vif_infos = vmwarevif.get_vif_info(self._session, [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] for vif in network_info: [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] return self._sync_wrapper(fn, *args, **kwargs) [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] self.wait() [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] self[:] = self._gt.wait() [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] return self._exit_event.wait() [ 702.746478] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] result = hub.switch() [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] return self.greenlet.switch() [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] result = function(*args, **kwargs) [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] return func(*args, **kwargs) [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] raise e [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] nwinfo = self.network_api.allocate_for_instance( [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 702.746811] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] created_port_ids = self._update_ports_for_instance( [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] with excutils.save_and_reraise_exception(): [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] self.force_reraise() [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] raise self.value [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] updated_port = self._update_port( [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] _ensure_no_port_binding_failure(port) [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 702.747163] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] raise exception.PortBindingFailed(port_id=port['id']) [ 702.747910] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] nova.exception.PortBindingFailed: Binding failed for port 89cee47c-37ae-4a0d-8184-0c7c53ff7bb1, please check neutron logs for more information. [ 702.747910] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] [ 702.747910] env[64020]: INFO nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Terminating instance [ 702.748279] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "refresh_cache-979aa8d0-1e9b-4bbb-b507-31f72542d8d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 702.748434] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquired lock "refresh_cache-979aa8d0-1e9b-4bbb-b507-31f72542d8d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 702.748597] env[64020]: DEBUG nova.network.neutron [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 702.749001] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 36945faa864d4cf48aa38941bd73b906 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 702.756689] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36945faa864d4cf48aa38941bd73b906 [ 702.852644] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.602s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 702.853162] env[64020]: DEBUG nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 702.854957] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg a2ec18a102454b378b001fe2df3b7b55 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 702.856448] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 18.770s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 702.856549] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 702.856695] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=64020) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 702.856966] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.605s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 702.858312] env[64020]: INFO nova.compute.claims [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 702.859823] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 628ffd0edc634c33b6a4445c30203b32 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 702.861571] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-931063bf-5acc-46bf-9439-4683deb0d465 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.869788] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac780b07-69ee-4973-922b-12a87c3b6134 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.874026] env[64020]: INFO nova.compute.manager [-] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Took 1.05 seconds to deallocate network for instance. [ 702.876933] env[64020]: DEBUG nova.compute.claims [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 702.877271] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 702.886344] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-060d8502-0699-4502-94a0-d1d668aeccf0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.898346] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0729f16-a029-4688-a958-68904ebcdc84 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.926616] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181790MB free_disk=120GB free_vcpus=48 pci_devices=None {{(pid=64020) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 702.926771] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 702.927564] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2ec18a102454b378b001fe2df3b7b55 [ 702.928102] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 628ffd0edc634c33b6a4445c30203b32 [ 703.266863] env[64020]: DEBUG nova.network.neutron [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 703.344815] env[64020]: DEBUG nova.network.neutron [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 703.345125] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 29715c23636940f5a3e83428a729b4c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 703.353820] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 29715c23636940f5a3e83428a729b4c8 [ 703.362635] env[64020]: DEBUG nova.compute.utils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 703.363096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg a0e875d19faf4e34a12dfeb2da7159af in queue reply_57893177120949e6a93cb88e15cd42b4 [ 703.363952] env[64020]: DEBUG nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 703.364146] env[64020]: DEBUG nova.network.neutron [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 703.368060] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 2ebb45f200f14976b0d5e2f97c8dc80c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 703.374525] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ebb45f200f14976b0d5e2f97c8dc80c [ 703.376765] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a0e875d19faf4e34a12dfeb2da7159af [ 703.406792] env[64020]: DEBUG nova.policy [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '29faa85bd7904cd985fb32f53b533c5b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '525215939f6741cea749e7d9e53f379e', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 703.543951] env[64020]: DEBUG nova.compute.manager [req-a6f43556-c825-417c-b789-ebed81184d28 req-a22575e0-0404-4932-b6f8-c47f34151224 service nova] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Received event network-changed-89cee47c-37ae-4a0d-8184-0c7c53ff7bb1 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 703.544174] env[64020]: DEBUG nova.compute.manager [req-a6f43556-c825-417c-b789-ebed81184d28 req-a22575e0-0404-4932-b6f8-c47f34151224 service nova] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Refreshing instance network info cache due to event network-changed-89cee47c-37ae-4a0d-8184-0c7c53ff7bb1. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 703.544424] env[64020]: DEBUG oslo_concurrency.lockutils [req-a6f43556-c825-417c-b789-ebed81184d28 req-a22575e0-0404-4932-b6f8-c47f34151224 service nova] Acquiring lock "refresh_cache-979aa8d0-1e9b-4bbb-b507-31f72542d8d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 703.687150] env[64020]: DEBUG nova.network.neutron [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Successfully created port: aa2cb781-03b3-4ff9-a6ae-989b384497fb {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 703.852791] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Releasing lock "refresh_cache-979aa8d0-1e9b-4bbb-b507-31f72542d8d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 703.852791] env[64020]: DEBUG nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 703.852791] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 703.852791] env[64020]: DEBUG oslo_concurrency.lockutils [req-a6f43556-c825-417c-b789-ebed81184d28 req-a22575e0-0404-4932-b6f8-c47f34151224 service nova] Acquired lock "refresh_cache-979aa8d0-1e9b-4bbb-b507-31f72542d8d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 703.852791] env[64020]: DEBUG nova.network.neutron [req-a6f43556-c825-417c-b789-ebed81184d28 req-a22575e0-0404-4932-b6f8-c47f34151224 service nova] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Refreshing network info cache for port 89cee47c-37ae-4a0d-8184-0c7c53ff7bb1 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 703.853184] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-a6f43556-c825-417c-b789-ebed81184d28 req-a22575e0-0404-4932-b6f8-c47f34151224 service nova] Expecting reply to msg 30f92aa19017440ca3ca9b08228bc2a5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 703.853184] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a0953b1c-2477-4f62-8c56-f3ee1489bb10 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.857324] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30f92aa19017440ca3ca9b08228bc2a5 [ 703.860197] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d265c42a-b6a6-42e3-bf27-fa157ec047f1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.871278] env[64020]: DEBUG nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 703.873171] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg c3e82f9b75804a93b07567dc3dc7e729 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 703.889991] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 979aa8d0-1e9b-4bbb-b507-31f72542d8d0 could not be found. [ 703.890238] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 703.890486] env[64020]: INFO nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Took 0.04 seconds to destroy the instance on the hypervisor. [ 703.890766] env[64020]: DEBUG oslo.service.loopingcall [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 703.891763] env[64020]: DEBUG nova.compute.manager [-] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 703.891855] env[64020]: DEBUG nova.network.neutron [-] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 703.914117] env[64020]: DEBUG nova.network.neutron [-] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 703.914621] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1e94e704acfa49869e15a3c77d7ead63 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 703.921521] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e94e704acfa49869e15a3c77d7ead63 [ 703.961318] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3e82f9b75804a93b07567dc3dc7e729 [ 704.312506] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5a6e918-ec18-432d-b6c0-f94d4a9242ba {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.321899] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fda3fe9f-1ee1-47fa-b59a-680f4debcd44 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.359800] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5c45cb5f-22fb-4b8b-92cb-58f5f148943b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.367819] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2aae36df-63e3-457b-8ba7-257031f80799 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.385499] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg f23a0a74d1f54a9b99ade4dfa5de376c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 704.387070] env[64020]: DEBUG nova.compute.provider_tree [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 704.387529] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 962b001393034d48b86f46d5fa366a4f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 704.389075] env[64020]: DEBUG nova.network.neutron [req-a6f43556-c825-417c-b789-ebed81184d28 req-a22575e0-0404-4932-b6f8-c47f34151224 service nova] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 704.398433] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 962b001393034d48b86f46d5fa366a4f [ 704.417156] env[64020]: DEBUG nova.network.neutron [-] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 704.418846] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1b0512edd36240dfae17c7e33568695e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 704.426157] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f23a0a74d1f54a9b99ade4dfa5de376c [ 704.430648] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b0512edd36240dfae17c7e33568695e [ 704.484349] env[64020]: DEBUG nova.network.neutron [req-a6f43556-c825-417c-b789-ebed81184d28 req-a22575e0-0404-4932-b6f8-c47f34151224 service nova] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 704.484853] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-a6f43556-c825-417c-b789-ebed81184d28 req-a22575e0-0404-4932-b6f8-c47f34151224 service nova] Expecting reply to msg 00a37ecfbb724522b27c75bd28e504e6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 704.493122] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 00a37ecfbb724522b27c75bd28e504e6 [ 704.737885] env[64020]: ERROR nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port aa2cb781-03b3-4ff9-a6ae-989b384497fb, please check neutron logs for more information. [ 704.737885] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 704.737885] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 704.737885] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 704.737885] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 704.737885] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 704.737885] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 704.737885] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 704.737885] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 704.737885] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 704.737885] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 704.737885] env[64020]: ERROR nova.compute.manager raise self.value [ 704.737885] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 704.737885] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 704.737885] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 704.737885] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 704.738366] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 704.738366] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 704.738366] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port aa2cb781-03b3-4ff9-a6ae-989b384497fb, please check neutron logs for more information. [ 704.738366] env[64020]: ERROR nova.compute.manager [ 704.738366] env[64020]: Traceback (most recent call last): [ 704.738366] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 704.738366] env[64020]: listener.cb(fileno) [ 704.738366] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 704.738366] env[64020]: result = function(*args, **kwargs) [ 704.738366] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 704.738366] env[64020]: return func(*args, **kwargs) [ 704.738366] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 704.738366] env[64020]: raise e [ 704.738366] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 704.738366] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 704.738366] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 704.738366] env[64020]: created_port_ids = self._update_ports_for_instance( [ 704.738366] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 704.738366] env[64020]: with excutils.save_and_reraise_exception(): [ 704.738366] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 704.738366] env[64020]: self.force_reraise() [ 704.738366] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 704.738366] env[64020]: raise self.value [ 704.738366] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 704.738366] env[64020]: updated_port = self._update_port( [ 704.738366] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 704.738366] env[64020]: _ensure_no_port_binding_failure(port) [ 704.738366] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 704.738366] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 704.739216] env[64020]: nova.exception.PortBindingFailed: Binding failed for port aa2cb781-03b3-4ff9-a6ae-989b384497fb, please check neutron logs for more information. [ 704.739216] env[64020]: Removing descriptor: 18 [ 704.890068] env[64020]: DEBUG nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 704.894079] env[64020]: DEBUG nova.scheduler.client.report [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 704.896594] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 44faa63c5a614d239b3157cc492de9fe in queue reply_57893177120949e6a93cb88e15cd42b4 [ 704.914779] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44faa63c5a614d239b3157cc492de9fe [ 704.917218] env[64020]: DEBUG nova.virt.hardware [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 704.917438] env[64020]: DEBUG nova.virt.hardware [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 704.917591] env[64020]: DEBUG nova.virt.hardware [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 704.917768] env[64020]: DEBUG nova.virt.hardware [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 704.917909] env[64020]: DEBUG nova.virt.hardware [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 704.918049] env[64020]: DEBUG nova.virt.hardware [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 704.918254] env[64020]: DEBUG nova.virt.hardware [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 704.918403] env[64020]: DEBUG nova.virt.hardware [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 704.918569] env[64020]: DEBUG nova.virt.hardware [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 704.918725] env[64020]: DEBUG nova.virt.hardware [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 704.918892] env[64020]: DEBUG nova.virt.hardware [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 704.920023] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fcd23259-5fd7-4489-a806-6205845d8d66 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.923464] env[64020]: INFO nova.compute.manager [-] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Took 1.03 seconds to deallocate network for instance. [ 704.928219] env[64020]: DEBUG nova.compute.claims [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 704.928390] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 704.929598] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-043d2e9e-3bd8-434c-b802-b05e82ce57e9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.944937] env[64020]: ERROR nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port aa2cb781-03b3-4ff9-a6ae-989b384497fb, please check neutron logs for more information. [ 704.944937] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Traceback (most recent call last): [ 704.944937] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 704.944937] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] yield resources [ 704.944937] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 704.944937] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] self.driver.spawn(context, instance, image_meta, [ 704.944937] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 704.944937] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 704.944937] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 704.944937] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] vm_ref = self.build_virtual_machine(instance, [ 704.944937] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] vif_infos = vmwarevif.get_vif_info(self._session, [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] for vif in network_info: [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] return self._sync_wrapper(fn, *args, **kwargs) [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] self.wait() [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] self[:] = self._gt.wait() [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] return self._exit_event.wait() [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 704.945397] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] current.throw(*self._exc) [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] result = function(*args, **kwargs) [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] return func(*args, **kwargs) [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] raise e [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] nwinfo = self.network_api.allocate_for_instance( [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] created_port_ids = self._update_ports_for_instance( [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] with excutils.save_and_reraise_exception(): [ 704.945816] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 704.946187] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] self.force_reraise() [ 704.946187] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 704.946187] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] raise self.value [ 704.946187] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 704.946187] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] updated_port = self._update_port( [ 704.946187] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 704.946187] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] _ensure_no_port_binding_failure(port) [ 704.946187] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 704.946187] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] raise exception.PortBindingFailed(port_id=port['id']) [ 704.946187] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] nova.exception.PortBindingFailed: Binding failed for port aa2cb781-03b3-4ff9-a6ae-989b384497fb, please check neutron logs for more information. [ 704.946187] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] [ 704.946187] env[64020]: INFO nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Terminating instance [ 704.947017] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "refresh_cache-0d97356d-3abf-4207-842d-6e2d468ef97c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 704.947168] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquired lock "refresh_cache-0d97356d-3abf-4207-842d-6e2d468ef97c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 704.947324] env[64020]: DEBUG nova.network.neutron [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 704.947778] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 7918c40efa1b4979b8aefda833235a80 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 704.954264] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7918c40efa1b4979b8aefda833235a80 [ 704.994103] env[64020]: DEBUG oslo_concurrency.lockutils [req-a6f43556-c825-417c-b789-ebed81184d28 req-a22575e0-0404-4932-b6f8-c47f34151224 service nova] Releasing lock "refresh_cache-979aa8d0-1e9b-4bbb-b507-31f72542d8d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 704.994293] env[64020]: DEBUG nova.compute.manager [req-a6f43556-c825-417c-b789-ebed81184d28 req-a22575e0-0404-4932-b6f8-c47f34151224 service nova] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Received event network-vif-deleted-89cee47c-37ae-4a0d-8184-0c7c53ff7bb1 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 705.399835] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.543s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 705.400399] env[64020]: DEBUG nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 705.402276] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg c0d61bc48c32485f829a40215b0e5969 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 705.408049] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.114s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 705.409978] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg 47105647da8b44d792ad59f8270f449b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 705.448469] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0d61bc48c32485f829a40215b0e5969 [ 705.454960] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47105647da8b44d792ad59f8270f449b [ 705.469089] env[64020]: DEBUG nova.network.neutron [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 705.564972] env[64020]: DEBUG nova.network.neutron [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 705.565491] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 19ecea48221f4a5da3830af8dac54fc7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 705.575687] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19ecea48221f4a5da3830af8dac54fc7 [ 705.643158] env[64020]: DEBUG nova.compute.manager [req-8fc6c106-caa8-4b74-ac1c-7877bb426905 req-200069a8-9a80-42a7-9946-2bf523efd79b service nova] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Received event network-changed-aa2cb781-03b3-4ff9-a6ae-989b384497fb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 705.643479] env[64020]: DEBUG nova.compute.manager [req-8fc6c106-caa8-4b74-ac1c-7877bb426905 req-200069a8-9a80-42a7-9946-2bf523efd79b service nova] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Refreshing instance network info cache due to event network-changed-aa2cb781-03b3-4ff9-a6ae-989b384497fb. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 705.643691] env[64020]: DEBUG oslo_concurrency.lockutils [req-8fc6c106-caa8-4b74-ac1c-7877bb426905 req-200069a8-9a80-42a7-9946-2bf523efd79b service nova] Acquiring lock "refresh_cache-0d97356d-3abf-4207-842d-6e2d468ef97c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 705.909743] env[64020]: DEBUG nova.compute.utils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 705.910388] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg dfe5a989ad604ecbb276536a3dc1f93a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 705.932475] env[64020]: DEBUG nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 705.932475] env[64020]: DEBUG nova.network.neutron [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 705.932475] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dfe5a989ad604ecbb276536a3dc1f93a [ 705.969923] env[64020]: DEBUG nova.policy [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '45a46e9b318d4026904493453f6611f0', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'dd7b26874a124d2386c032cc7f72728c', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 706.067791] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Releasing lock "refresh_cache-0d97356d-3abf-4207-842d-6e2d468ef97c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 706.068250] env[64020]: DEBUG nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 706.068445] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 706.068751] env[64020]: DEBUG oslo_concurrency.lockutils [req-8fc6c106-caa8-4b74-ac1c-7877bb426905 req-200069a8-9a80-42a7-9946-2bf523efd79b service nova] Acquired lock "refresh_cache-0d97356d-3abf-4207-842d-6e2d468ef97c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 706.068917] env[64020]: DEBUG nova.network.neutron [req-8fc6c106-caa8-4b74-ac1c-7877bb426905 req-200069a8-9a80-42a7-9946-2bf523efd79b service nova] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Refreshing network info cache for port aa2cb781-03b3-4ff9-a6ae-989b384497fb {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 706.069372] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-8fc6c106-caa8-4b74-ac1c-7877bb426905 req-200069a8-9a80-42a7-9946-2bf523efd79b service nova] Expecting reply to msg d41a47bf33cc4620b5d4916eee208954 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 706.070135] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1f0d3a59-394c-4b26-973f-aafc18c8668f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.076759] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d41a47bf33cc4620b5d4916eee208954 [ 706.082562] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4c15338d-9378-434c-81e6-87ebba636d25 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.114040] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0d97356d-3abf-4207-842d-6e2d468ef97c could not be found. [ 706.114783] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 706.114783] env[64020]: INFO nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Took 0.05 seconds to destroy the instance on the hypervisor. [ 706.114783] env[64020]: DEBUG oslo.service.loopingcall [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 706.117955] env[64020]: DEBUG nova.compute.manager [-] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 706.118085] env[64020]: DEBUG nova.network.neutron [-] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 706.154384] env[64020]: DEBUG nova.network.neutron [-] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 706.154935] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 9c7d2fe81e6442189fb8714ea25ce0de in queue reply_57893177120949e6a93cb88e15cd42b4 [ 706.165628] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c7d2fe81e6442189fb8714ea25ce0de [ 706.275869] env[64020]: DEBUG nova.network.neutron [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Successfully created port: 00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 706.418031] env[64020]: DEBUG nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 706.418031] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg afc7e63af86d4ee0a6ff4447adf46156 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 706.465390] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg afc7e63af86d4ee0a6ff4447adf46156 [ 706.503941] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94dda0c4-5fc5-43f1-8a55-39aa1cb42fff {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.511607] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0fa5bfee-6a2b-43b0-83fb-4d4153f964f3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.553369] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-022ebd56-9d9e-43df-8f1d-6c7f1a1c5b2f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.561962] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2baf6ae4-651c-45c0-b0ab-3f84bf6fe37d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.579224] env[64020]: DEBUG nova.compute.provider_tree [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 706.579797] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg 20b194a68c5e4ba19a29147e64a43bac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 706.588262] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20b194a68c5e4ba19a29147e64a43bac [ 706.606283] env[64020]: DEBUG nova.network.neutron [req-8fc6c106-caa8-4b74-ac1c-7877bb426905 req-200069a8-9a80-42a7-9946-2bf523efd79b service nova] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 706.661078] env[64020]: DEBUG nova.network.neutron [-] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 706.661807] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg e7b77cc7f3ba4908a35fbe39a91a87bb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 706.671570] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e7b77cc7f3ba4908a35fbe39a91a87bb [ 706.841175] env[64020]: DEBUG nova.network.neutron [req-8fc6c106-caa8-4b74-ac1c-7877bb426905 req-200069a8-9a80-42a7-9946-2bf523efd79b service nova] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 706.841720] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-8fc6c106-caa8-4b74-ac1c-7877bb426905 req-200069a8-9a80-42a7-9946-2bf523efd79b service nova] Expecting reply to msg 8b2f702130d5455cb02bc9314cd36751 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 706.849347] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b2f702130d5455cb02bc9314cd36751 [ 706.923526] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 26e91fcf916042e8a0d09ef7636c48ce in queue reply_57893177120949e6a93cb88e15cd42b4 [ 706.961135] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26e91fcf916042e8a0d09ef7636c48ce [ 707.083923] env[64020]: DEBUG nova.scheduler.client.report [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 707.086701] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg c45d08cde0bd46ffa98261de001be961 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 707.101942] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c45d08cde0bd46ffa98261de001be961 [ 707.172420] env[64020]: INFO nova.compute.manager [-] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Took 1.05 seconds to deallocate network for instance. [ 707.174599] env[64020]: DEBUG nova.compute.claims [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 707.174768] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 707.189992] env[64020]: ERROR nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4, please check neutron logs for more information. [ 707.189992] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 707.189992] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 707.189992] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 707.189992] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 707.189992] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 707.189992] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 707.189992] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 707.189992] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 707.189992] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 707.189992] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 707.189992] env[64020]: ERROR nova.compute.manager raise self.value [ 707.189992] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 707.189992] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 707.189992] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 707.189992] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 707.190425] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 707.190425] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 707.190425] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4, please check neutron logs for more information. [ 707.190425] env[64020]: ERROR nova.compute.manager [ 707.190425] env[64020]: Traceback (most recent call last): [ 707.190425] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 707.190425] env[64020]: listener.cb(fileno) [ 707.190425] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 707.190425] env[64020]: result = function(*args, **kwargs) [ 707.190425] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 707.190425] env[64020]: return func(*args, **kwargs) [ 707.190425] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 707.190425] env[64020]: raise e [ 707.190425] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 707.190425] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 707.190425] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 707.190425] env[64020]: created_port_ids = self._update_ports_for_instance( [ 707.190425] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 707.190425] env[64020]: with excutils.save_and_reraise_exception(): [ 707.190425] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 707.190425] env[64020]: self.force_reraise() [ 707.190425] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 707.190425] env[64020]: raise self.value [ 707.190425] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 707.190425] env[64020]: updated_port = self._update_port( [ 707.190425] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 707.190425] env[64020]: _ensure_no_port_binding_failure(port) [ 707.190425] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 707.190425] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 707.191642] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4, please check neutron logs for more information. [ 707.191642] env[64020]: Removing descriptor: 18 [ 707.344207] env[64020]: DEBUG oslo_concurrency.lockutils [req-8fc6c106-caa8-4b74-ac1c-7877bb426905 req-200069a8-9a80-42a7-9946-2bf523efd79b service nova] Releasing lock "refresh_cache-0d97356d-3abf-4207-842d-6e2d468ef97c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 707.344489] env[64020]: DEBUG nova.compute.manager [req-8fc6c106-caa8-4b74-ac1c-7877bb426905 req-200069a8-9a80-42a7-9946-2bf523efd79b service nova] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Received event network-vif-deleted-aa2cb781-03b3-4ff9-a6ae-989b384497fb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 707.431131] env[64020]: DEBUG nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 707.455936] env[64020]: DEBUG nova.virt.hardware [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 707.456188] env[64020]: DEBUG nova.virt.hardware [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 707.456337] env[64020]: DEBUG nova.virt.hardware [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 707.456512] env[64020]: DEBUG nova.virt.hardware [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 707.456650] env[64020]: DEBUG nova.virt.hardware [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 707.456788] env[64020]: DEBUG nova.virt.hardware [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 707.456985] env[64020]: DEBUG nova.virt.hardware [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 707.457143] env[64020]: DEBUG nova.virt.hardware [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 707.457310] env[64020]: DEBUG nova.virt.hardware [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 707.457457] env[64020]: DEBUG nova.virt.hardware [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 707.457718] env[64020]: DEBUG nova.virt.hardware [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 707.458629] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-346f97bf-e9ad-45e0-8a8d-103afeb7a70b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 707.469056] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e72bb050-df86-4b36-b404-bcb93bfa527f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 707.484427] env[64020]: ERROR nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4, please check neutron logs for more information. [ 707.484427] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Traceback (most recent call last): [ 707.484427] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 707.484427] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] yield resources [ 707.484427] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 707.484427] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] self.driver.spawn(context, instance, image_meta, [ 707.484427] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 707.484427] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 707.484427] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 707.484427] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] vm_ref = self.build_virtual_machine(instance, [ 707.484427] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] vif_infos = vmwarevif.get_vif_info(self._session, [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] for vif in network_info: [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] return self._sync_wrapper(fn, *args, **kwargs) [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] self.wait() [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] self[:] = self._gt.wait() [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] return self._exit_event.wait() [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 707.484703] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] current.throw(*self._exc) [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] result = function(*args, **kwargs) [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] return func(*args, **kwargs) [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] raise e [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] nwinfo = self.network_api.allocate_for_instance( [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] created_port_ids = self._update_ports_for_instance( [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] with excutils.save_and_reraise_exception(): [ 707.485005] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 707.485314] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] self.force_reraise() [ 707.485314] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 707.485314] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] raise self.value [ 707.485314] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 707.485314] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] updated_port = self._update_port( [ 707.485314] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 707.485314] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] _ensure_no_port_binding_failure(port) [ 707.485314] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 707.485314] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] raise exception.PortBindingFailed(port_id=port['id']) [ 707.485314] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] nova.exception.PortBindingFailed: Binding failed for port 00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4, please check neutron logs for more information. [ 707.485314] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] [ 707.485314] env[64020]: INFO nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Terminating instance [ 707.486991] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Acquiring lock "refresh_cache-727e7787-68b7-441c-a31d-6f0796c33be0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 707.487146] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Acquired lock "refresh_cache-727e7787-68b7-441c-a31d-6f0796c33be0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 707.487304] env[64020]: DEBUG nova.network.neutron [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 707.487720] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg d8ef9a4be45e49a3b2700baa7c2b3edd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 707.504918] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d8ef9a4be45e49a3b2700baa7c2b3edd [ 707.589173] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.181s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 707.589816] env[64020]: ERROR nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e1f39557-dca1-4318-9e14-86de0b534ae7, please check neutron logs for more information. [ 707.589816] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Traceback (most recent call last): [ 707.589816] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 707.589816] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] self.driver.spawn(context, instance, image_meta, [ 707.589816] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 707.589816] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] self._vmops.spawn(context, instance, image_meta, injected_files, [ 707.589816] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 707.589816] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] vm_ref = self.build_virtual_machine(instance, [ 707.589816] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 707.589816] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] vif_infos = vmwarevif.get_vif_info(self._session, [ 707.589816] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] for vif in network_info: [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] return self._sync_wrapper(fn, *args, **kwargs) [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] self.wait() [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] self[:] = self._gt.wait() [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] return self._exit_event.wait() [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] result = hub.switch() [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 707.590163] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] return self.greenlet.switch() [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] result = function(*args, **kwargs) [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] return func(*args, **kwargs) [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] raise e [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] nwinfo = self.network_api.allocate_for_instance( [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] created_port_ids = self._update_ports_for_instance( [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] with excutils.save_and_reraise_exception(): [ 707.590534] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 707.590900] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] self.force_reraise() [ 707.590900] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 707.590900] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] raise self.value [ 707.590900] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 707.590900] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] updated_port = self._update_port( [ 707.590900] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 707.590900] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] _ensure_no_port_binding_failure(port) [ 707.590900] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 707.590900] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] raise exception.PortBindingFailed(port_id=port['id']) [ 707.590900] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] nova.exception.PortBindingFailed: Binding failed for port e1f39557-dca1-4318-9e14-86de0b534ae7, please check neutron logs for more information. [ 707.590900] env[64020]: ERROR nova.compute.manager [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] [ 707.591227] env[64020]: DEBUG nova.compute.utils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Binding failed for port e1f39557-dca1-4318-9e14-86de0b534ae7, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 707.591752] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.572s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 707.593616] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 2617bc1a660a46d6ad88a2d28883ab5b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 707.595167] env[64020]: DEBUG nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Build of instance 5ac25e1d-007a-4eaa-811c-655b54efd7ec was re-scheduled: Binding failed for port e1f39557-dca1-4318-9e14-86de0b534ae7, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 707.595590] env[64020]: DEBUG nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 707.596670] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Acquiring lock "refresh_cache-5ac25e1d-007a-4eaa-811c-655b54efd7ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 707.596670] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Acquired lock "refresh_cache-5ac25e1d-007a-4eaa-811c-655b54efd7ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 707.596670] env[64020]: DEBUG nova.network.neutron [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 707.597084] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg 3731793e42594fdea659b75098ef6a6f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 707.604556] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3731793e42594fdea659b75098ef6a6f [ 707.629188] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2617bc1a660a46d6ad88a2d28883ab5b [ 707.736491] env[64020]: DEBUG nova.compute.manager [req-2a60bc6a-22c9-4d0e-938a-21e59556c161 req-c579b369-cde6-4179-af70-70ef58f89d5f service nova] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Received event network-changed-00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 707.736686] env[64020]: DEBUG nova.compute.manager [req-2a60bc6a-22c9-4d0e-938a-21e59556c161 req-c579b369-cde6-4179-af70-70ef58f89d5f service nova] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Refreshing instance network info cache due to event network-changed-00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 707.736867] env[64020]: DEBUG oslo_concurrency.lockutils [req-2a60bc6a-22c9-4d0e-938a-21e59556c161 req-c579b369-cde6-4179-af70-70ef58f89d5f service nova] Acquiring lock "refresh_cache-727e7787-68b7-441c-a31d-6f0796c33be0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 708.012622] env[64020]: DEBUG nova.network.neutron [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 708.103914] env[64020]: DEBUG nova.network.neutron [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 708.104982] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 43f41ddc471e48d8ae2a59f0a3a9b03d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 708.113052] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43f41ddc471e48d8ae2a59f0a3a9b03d [ 708.141043] env[64020]: DEBUG nova.network.neutron [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 708.298781] env[64020]: DEBUG nova.network.neutron [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 708.299300] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg ff954d66dd7b4b8cb2b814dec9e33b29 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 708.311655] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff954d66dd7b4b8cb2b814dec9e33b29 [ 708.531758] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76f34ece-2967-4ff6-baf3-907a3e840360 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.539664] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11d349a4-adae-43cc-90df-46de10fdec57 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.568466] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ade3e678-2c49-4476-850f-a8bf3d852456 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.575835] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c75e6ac6-b249-4530-8975-feb97049194a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.589552] env[64020]: DEBUG nova.compute.provider_tree [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 708.590195] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 626d4bd2a8dc417ca522160a752a393a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 708.597543] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 626d4bd2a8dc417ca522160a752a393a [ 708.609604] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Releasing lock "refresh_cache-727e7787-68b7-441c-a31d-6f0796c33be0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 708.609604] env[64020]: DEBUG nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 708.609743] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 708.609974] env[64020]: DEBUG oslo_concurrency.lockutils [req-2a60bc6a-22c9-4d0e-938a-21e59556c161 req-c579b369-cde6-4179-af70-70ef58f89d5f service nova] Acquired lock "refresh_cache-727e7787-68b7-441c-a31d-6f0796c33be0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 708.610163] env[64020]: DEBUG nova.network.neutron [req-2a60bc6a-22c9-4d0e-938a-21e59556c161 req-c579b369-cde6-4179-af70-70ef58f89d5f service nova] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Refreshing network info cache for port 00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 708.610610] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-2a60bc6a-22c9-4d0e-938a-21e59556c161 req-c579b369-cde6-4179-af70-70ef58f89d5f service nova] Expecting reply to msg 2c48f439c3f14d88b3ed42e3f8782847 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 708.611458] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-308b5de9-c6ab-4b80-b5e9-a4e5bc2476e9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.618524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c48f439c3f14d88b3ed42e3f8782847 [ 708.621731] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ba8423d-7fe3-4717-b1e5-8d58ef395757 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.645345] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 727e7787-68b7-441c-a31d-6f0796c33be0 could not be found. [ 708.645608] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 708.645998] env[64020]: INFO nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Took 0.04 seconds to destroy the instance on the hypervisor. [ 708.646254] env[64020]: DEBUG oslo.service.loopingcall [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 708.646462] env[64020]: DEBUG nova.compute.manager [-] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 708.646648] env[64020]: DEBUG nova.network.neutron [-] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 708.801375] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Releasing lock "refresh_cache-5ac25e1d-007a-4eaa-811c-655b54efd7ec" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 708.801636] env[64020]: DEBUG nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 708.801806] env[64020]: DEBUG nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 708.801969] env[64020]: DEBUG nova.network.neutron [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 708.820453] env[64020]: DEBUG nova.network.neutron [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 708.821152] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg 848ebb59e3f44029a9ec0f83c2d4e170 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 708.828157] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 848ebb59e3f44029a9ec0f83c2d4e170 [ 708.838538] env[64020]: DEBUG nova.network.neutron [-] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 708.839406] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 82e2b5e1b4364018bde9469dc90519d5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 708.845217] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82e2b5e1b4364018bde9469dc90519d5 [ 709.093464] env[64020]: DEBUG nova.scheduler.client.report [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 709.095891] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg aa9fbb9f0d134ca7a0e5df7b2e1c9b6f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 709.111434] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa9fbb9f0d134ca7a0e5df7b2e1c9b6f [ 709.140346] env[64020]: DEBUG nova.network.neutron [req-2a60bc6a-22c9-4d0e-938a-21e59556c161 req-c579b369-cde6-4179-af70-70ef58f89d5f service nova] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 709.267280] env[64020]: DEBUG nova.network.neutron [req-2a60bc6a-22c9-4d0e-938a-21e59556c161 req-c579b369-cde6-4179-af70-70ef58f89d5f service nova] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 709.267828] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-2a60bc6a-22c9-4d0e-938a-21e59556c161 req-c579b369-cde6-4179-af70-70ef58f89d5f service nova] Expecting reply to msg fd0f0addfa664d679dbdc3509f1dc895 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 709.276986] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd0f0addfa664d679dbdc3509f1dc895 [ 709.323577] env[64020]: DEBUG nova.network.neutron [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 709.324109] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg f0af6587fd134c33b63429622cc439a4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 709.331952] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f0af6587fd134c33b63429622cc439a4 [ 709.341488] env[64020]: DEBUG nova.network.neutron [-] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 709.341870] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 48a4919bbdf344b5a4bbc81900da2e2f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 709.349628] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 48a4919bbdf344b5a4bbc81900da2e2f [ 709.600026] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.007s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 709.600026] env[64020]: ERROR nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b7cef9b2-89e8-4bfc-8a6b-fe379e46900f, please check neutron logs for more information. [ 709.600026] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Traceback (most recent call last): [ 709.600026] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 709.600026] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] self.driver.spawn(context, instance, image_meta, [ 709.600026] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 709.600026] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 709.600026] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 709.600026] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] vm_ref = self.build_virtual_machine(instance, [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] vif_infos = vmwarevif.get_vif_info(self._session, [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] for vif in network_info: [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] return self._sync_wrapper(fn, *args, **kwargs) [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] self.wait() [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] self[:] = self._gt.wait() [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] return self._exit_event.wait() [ 709.600562] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] result = hub.switch() [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] return self.greenlet.switch() [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] result = function(*args, **kwargs) [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] return func(*args, **kwargs) [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] raise e [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] nwinfo = self.network_api.allocate_for_instance( [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 709.600912] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] created_port_ids = self._update_ports_for_instance( [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] with excutils.save_and_reraise_exception(): [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] self.force_reraise() [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] raise self.value [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] updated_port = self._update_port( [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] _ensure_no_port_binding_failure(port) [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 709.601283] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] raise exception.PortBindingFailed(port_id=port['id']) [ 709.601600] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] nova.exception.PortBindingFailed: Binding failed for port b7cef9b2-89e8-4bfc-8a6b-fe379e46900f, please check neutron logs for more information. [ 709.601600] env[64020]: ERROR nova.compute.manager [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] [ 709.601600] env[64020]: DEBUG nova.compute.utils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Binding failed for port b7cef9b2-89e8-4bfc-8a6b-fe379e46900f, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 709.601600] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.742s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 709.603375] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 4395debbf4114038b9c4e78b95d67b4f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 709.604649] env[64020]: DEBUG nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Build of instance d4d0d50d-8479-47f7-ba29-114ef10764cc was re-scheduled: Binding failed for port b7cef9b2-89e8-4bfc-8a6b-fe379e46900f, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 709.605074] env[64020]: DEBUG nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 709.605288] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-d4d0d50d-8479-47f7-ba29-114ef10764cc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 709.605430] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-d4d0d50d-8479-47f7-ba29-114ef10764cc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 709.605584] env[64020]: DEBUG nova.network.neutron [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 709.605954] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg cfba874736e6464ea710f00757529c0f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 709.617520] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfba874736e6464ea710f00757529c0f [ 709.644350] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquiring lock "d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 709.644584] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 709.645245] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4395debbf4114038b9c4e78b95d67b4f [ 709.770448] env[64020]: DEBUG oslo_concurrency.lockutils [req-2a60bc6a-22c9-4d0e-938a-21e59556c161 req-c579b369-cde6-4179-af70-70ef58f89d5f service nova] Releasing lock "refresh_cache-727e7787-68b7-441c-a31d-6f0796c33be0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 709.770719] env[64020]: DEBUG nova.compute.manager [req-2a60bc6a-22c9-4d0e-938a-21e59556c161 req-c579b369-cde6-4179-af70-70ef58f89d5f service nova] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Received event network-vif-deleted-00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 709.826655] env[64020]: INFO nova.compute.manager [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] [instance: 5ac25e1d-007a-4eaa-811c-655b54efd7ec] Took 1.02 seconds to deallocate network for instance. [ 709.828434] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg e3a58b1c437743f3b4b5a91a47020aac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 709.844433] env[64020]: INFO nova.compute.manager [-] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Took 1.20 seconds to deallocate network for instance. [ 709.846756] env[64020]: DEBUG nova.compute.claims [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 709.846928] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 709.863578] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3a58b1c437743f3b4b5a91a47020aac [ 710.137625] env[64020]: DEBUG nova.network.neutron [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 710.249187] env[64020]: DEBUG nova.network.neutron [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 710.249699] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 2c34be9058c541979b452819d40a2dc4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 710.260553] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c34be9058c541979b452819d40a2dc4 [ 710.334358] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg ad12a2f49a9b4de2b266d10dbfc7fbc8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 710.372705] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad12a2f49a9b4de2b266d10dbfc7fbc8 [ 710.507799] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a01e2b9d-43a4-4326-976c-a420050e1a08 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 710.515160] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca4c64ac-03b8-4466-9e3d-fd6e3307ad31 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 710.545058] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8508e6e-b66c-4266-8ad3-e3f4f42bd2a2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 710.552152] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-523c9abc-3cc9-4344-9526-2524c8ca01a1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 710.564635] env[64020]: DEBUG nova.compute.provider_tree [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 710.565104] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 1de4c70e0e5648d8aed481f2857d5602 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 710.572667] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1de4c70e0e5648d8aed481f2857d5602 [ 710.752143] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-d4d0d50d-8479-47f7-ba29-114ef10764cc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 710.752461] env[64020]: DEBUG nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 710.752711] env[64020]: DEBUG nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 710.752819] env[64020]: DEBUG nova.network.neutron [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 710.766575] env[64020]: DEBUG nova.network.neutron [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 710.767141] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 4c993d26d21942549b568e8ccc41b922 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 710.774081] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c993d26d21942549b568e8ccc41b922 [ 710.859884] env[64020]: INFO nova.scheduler.client.report [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Deleted allocations for instance 5ac25e1d-007a-4eaa-811c-655b54efd7ec [ 710.866105] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Expecting reply to msg 7629f0940ae741ed9195481b6f7aba77 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 710.878493] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7629f0940ae741ed9195481b6f7aba77 [ 711.068032] env[64020]: DEBUG nova.scheduler.client.report [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 711.070634] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg b3f67074e68947e48ee1309cd31de489 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 711.081384] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3f67074e68947e48ee1309cd31de489 [ 711.269427] env[64020]: DEBUG nova.network.neutron [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 711.269949] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 84d21af3e19e46018eddabf7221c3745 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 711.277867] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 84d21af3e19e46018eddabf7221c3745 [ 711.368250] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d83f16f7-bdcc-4b54-a5fc-bded4e66c7a9 tempest-ImagesNegativeTestJSON-2060354573 tempest-ImagesNegativeTestJSON-2060354573-project-member] Lock "5ac25e1d-007a-4eaa-811c-655b54efd7ec" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 98.762s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 711.368851] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg c5d862534e914e64a62d78136d888381 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 711.377670] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5d862534e914e64a62d78136d888381 [ 711.573570] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.972s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 711.574213] env[64020]: ERROR nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 7d184f73-764c-4ec8-84ce-066b295da5fb, please check neutron logs for more information. [ 711.574213] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Traceback (most recent call last): [ 711.574213] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 711.574213] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] self.driver.spawn(context, instance, image_meta, [ 711.574213] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 711.574213] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] self._vmops.spawn(context, instance, image_meta, injected_files, [ 711.574213] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 711.574213] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] vm_ref = self.build_virtual_machine(instance, [ 711.574213] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 711.574213] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] vif_infos = vmwarevif.get_vif_info(self._session, [ 711.574213] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] for vif in network_info: [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] return self._sync_wrapper(fn, *args, **kwargs) [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] self.wait() [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] self[:] = self._gt.wait() [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] return self._exit_event.wait() [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] result = hub.switch() [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 711.574539] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] return self.greenlet.switch() [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] result = function(*args, **kwargs) [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] return func(*args, **kwargs) [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] raise e [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] nwinfo = self.network_api.allocate_for_instance( [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] created_port_ids = self._update_ports_for_instance( [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] with excutils.save_and_reraise_exception(): [ 711.574886] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 711.575217] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] self.force_reraise() [ 711.575217] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 711.575217] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] raise self.value [ 711.575217] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 711.575217] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] updated_port = self._update_port( [ 711.575217] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 711.575217] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] _ensure_no_port_binding_failure(port) [ 711.575217] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 711.575217] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] raise exception.PortBindingFailed(port_id=port['id']) [ 711.575217] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] nova.exception.PortBindingFailed: Binding failed for port 7d184f73-764c-4ec8-84ce-066b295da5fb, please check neutron logs for more information. [ 711.575217] env[64020]: ERROR nova.compute.manager [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] [ 711.575496] env[64020]: DEBUG nova.compute.utils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Binding failed for port 7d184f73-764c-4ec8-84ce-066b295da5fb, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 711.576115] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.395s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 711.577890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 713a7011852f4be6aec561a7829839c7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 711.580720] env[64020]: DEBUG nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Build of instance 7437f3e0-0112-4640-9076-5f5f864ec230 was re-scheduled: Binding failed for port 7d184f73-764c-4ec8-84ce-066b295da5fb, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 711.580818] env[64020]: DEBUG nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 711.581034] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquiring lock "refresh_cache-7437f3e0-0112-4640-9076-5f5f864ec230" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 711.581212] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquired lock "refresh_cache-7437f3e0-0112-4640-9076-5f5f864ec230" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 711.581374] env[64020]: DEBUG nova.network.neutron [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 711.581740] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg e45217e9e61c46d5a8255c2680c35d5d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 711.587803] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e45217e9e61c46d5a8255c2680c35d5d [ 711.614007] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 713a7011852f4be6aec561a7829839c7 [ 711.772166] env[64020]: INFO nova.compute.manager [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: d4d0d50d-8479-47f7-ba29-114ef10764cc] Took 1.02 seconds to deallocate network for instance. [ 711.774047] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 9158f17df9c94cd9885dfbe3f9f64d8d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 711.811517] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9158f17df9c94cd9885dfbe3f9f64d8d [ 711.870929] env[64020]: DEBUG nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 711.872721] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 25e8aed21f1741dfbda98fc37585877f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 711.909191] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 25e8aed21f1741dfbda98fc37585877f [ 712.114034] env[64020]: DEBUG nova.network.neutron [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 712.277965] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 4251e4c179bb4743968c69a3b738ad59 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 712.301990] env[64020]: DEBUG nova.network.neutron [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 712.302503] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg a4ebbed483b347ea95e5d5a52bfa53c9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 712.311282] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4ebbed483b347ea95e5d5a52bfa53c9 [ 712.322323] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4251e4c179bb4743968c69a3b738ad59 [ 712.402297] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 712.577233] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-650c1e6b-04da-4b4b-855d-3cdbf4251adc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 712.592116] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0ce8492-ec9e-42e9-ad0c-9193f9272ed5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 712.629233] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dc67c10-012e-475e-9468-000ecb0c896d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 712.637710] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87d7a6f9-2ec8-4794-a417-c4e85c10d071 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 712.652619] env[64020]: DEBUG nova.compute.provider_tree [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 712.653269] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 4138850b503c4abe9da5a4b32d38a8e1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 712.662860] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4138850b503c4abe9da5a4b32d38a8e1 [ 712.804492] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Releasing lock "refresh_cache-7437f3e0-0112-4640-9076-5f5f864ec230" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 712.804745] env[64020]: DEBUG nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 712.804781] env[64020]: DEBUG nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 712.804929] env[64020]: DEBUG nova.network.neutron [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 712.810139] env[64020]: INFO nova.scheduler.client.report [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Deleted allocations for instance d4d0d50d-8479-47f7-ba29-114ef10764cc [ 712.816383] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 8b617894a3e54b7fa6d467398da3e00c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 712.824540] env[64020]: DEBUG nova.network.neutron [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 712.825076] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg c86dde5d1d59481bae8350ea41cde2f3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 712.831619] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b617894a3e54b7fa6d467398da3e00c [ 712.839401] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c86dde5d1d59481bae8350ea41cde2f3 [ 713.159776] env[64020]: DEBUG nova.scheduler.client.report [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 713.159776] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg f982c5682a9344b1bb42a4f68868f417 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 713.176995] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f982c5682a9344b1bb42a4f68868f417 [ 713.320861] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ddeedf51-56e9-4d6d-b86a-4886d820d61b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "d4d0d50d-8479-47f7-ba29-114ef10764cc" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 98.899s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.320861] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg b8367d6fa1fb477197a96bc220bffa2b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 713.327816] env[64020]: DEBUG nova.network.neutron [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 713.327816] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 35b4847237ad4e29a48604d79b8da801 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 713.332659] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8367d6fa1fb477197a96bc220bffa2b [ 713.370052] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35b4847237ad4e29a48604d79b8da801 [ 713.672270] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.088s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.672270] env[64020]: ERROR nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9, please check neutron logs for more information. [ 713.672270] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Traceback (most recent call last): [ 713.672270] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 713.672270] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] self.driver.spawn(context, instance, image_meta, [ 713.672270] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 713.672270] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] self._vmops.spawn(context, instance, image_meta, injected_files, [ 713.672270] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 713.672270] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] vm_ref = self.build_virtual_machine(instance, [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] vif_infos = vmwarevif.get_vif_info(self._session, [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] for vif in network_info: [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] return self._sync_wrapper(fn, *args, **kwargs) [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] self.wait() [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] self[:] = self._gt.wait() [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] return self._exit_event.wait() [ 713.672659] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] result = hub.switch() [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] return self.greenlet.switch() [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] result = function(*args, **kwargs) [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] return func(*args, **kwargs) [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] raise e [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] nwinfo = self.network_api.allocate_for_instance( [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 713.672991] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] created_port_ids = self._update_ports_for_instance( [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] with excutils.save_and_reraise_exception(): [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] self.force_reraise() [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] raise self.value [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] updated_port = self._update_port( [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] _ensure_no_port_binding_failure(port) [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 713.673300] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] raise exception.PortBindingFailed(port_id=port['id']) [ 713.673652] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] nova.exception.PortBindingFailed: Binding failed for port 9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9, please check neutron logs for more information. [ 713.673652] env[64020]: ERROR nova.compute.manager [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] [ 713.673652] env[64020]: DEBUG nova.compute.utils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Binding failed for port 9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 713.673652] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.401s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 713.673652] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 0c54d94b6e914575838899ebcfbe8d6b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 713.673802] env[64020]: DEBUG nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Build of instance af8b2b66-c2d6-4bb6-a266-52cb82bac3be was re-scheduled: Binding failed for port 9ff8fed8-5e82-4dc8-bdcb-31dc26d781d9, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 713.673802] env[64020]: DEBUG nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 713.673802] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "refresh_cache-af8b2b66-c2d6-4bb6-a266-52cb82bac3be" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 713.673802] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquired lock "refresh_cache-af8b2b66-c2d6-4bb6-a266-52cb82bac3be" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 713.673950] env[64020]: DEBUG nova.network.neutron [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 713.673950] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg fd2ebdf13d854adfafd8193237101b0b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 713.691938] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd2ebdf13d854adfafd8193237101b0b [ 713.714316] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c54d94b6e914575838899ebcfbe8d6b [ 713.822076] env[64020]: DEBUG nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 713.823856] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg a1ce475aa11e45018449da97e78c4558 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 713.829716] env[64020]: INFO nova.compute.manager [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 7437f3e0-0112-4640-9076-5f5f864ec230] Took 1.02 seconds to deallocate network for instance. [ 713.831442] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 8a0b5aa59c304eeb8e5b2032218e346a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 713.857753] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1ce475aa11e45018449da97e78c4558 [ 713.870462] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a0b5aa59c304eeb8e5b2032218e346a [ 714.196101] env[64020]: DEBUG nova.network.neutron [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 714.295274] env[64020]: DEBUG nova.network.neutron [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 714.295787] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 47f82b2e833c4a2d8ac2be3678351d0f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 714.304502] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47f82b2e833c4a2d8ac2be3678351d0f [ 714.335624] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg b113b0abcb884d999b0420969005f3df in queue reply_57893177120949e6a93cb88e15cd42b4 [ 714.361162] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 714.411078] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b113b0abcb884d999b0420969005f3df [ 714.632080] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9716c341-f424-4c47-9682-d67169d100ba {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.638614] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ba2ba85-b354-41e9-84c1-5a27e73d6240 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.678930] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83c8716e-e077-414d-b4b3-989b5fe7f9b6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.690546] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd3d17cc-3e28-4ad6-81b0-327bb42b2fd4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.700595] env[64020]: DEBUG nova.compute.provider_tree [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 714.701404] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg b526a6723b31419cbca6c5385eff28bd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 714.709755] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b526a6723b31419cbca6c5385eff28bd [ 714.804792] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Releasing lock "refresh_cache-af8b2b66-c2d6-4bb6-a266-52cb82bac3be" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 714.805085] env[64020]: DEBUG nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 714.805312] env[64020]: DEBUG nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 714.805524] env[64020]: DEBUG nova.network.neutron [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 714.851835] env[64020]: DEBUG nova.network.neutron [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 714.852461] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 0e329f94a7bd44c4ba3b390d4c465e43 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 714.857595] env[64020]: INFO nova.scheduler.client.report [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Deleted allocations for instance 7437f3e0-0112-4640-9076-5f5f864ec230 [ 714.863422] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e329f94a7bd44c4ba3b390d4c465e43 [ 714.864134] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 88055cdd67954a1ea1c5120833006d74 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 714.876390] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88055cdd67954a1ea1c5120833006d74 [ 715.203574] env[64020]: DEBUG nova.scheduler.client.report [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 715.206184] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 1f850506abc7482496dc6b9416750e5d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 715.219886] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f850506abc7482496dc6b9416750e5d [ 715.236298] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "ecb7f281-1206-4693-845b-c59f639b0789" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 715.236510] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "ecb7f281-1206-4693-845b-c59f639b0789" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 715.354288] env[64020]: DEBUG nova.network.neutron [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 715.354829] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg efeba98bf9e14fab88464185efe7f144 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 715.364157] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg efeba98bf9e14fab88464185efe7f144 [ 715.365641] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9863a6a9-919f-49d2-8705-abbd944cad08 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "7437f3e0-0112-4640-9076-5f5f864ec230" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 97.161s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 715.366451] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 9f0dba2afa07479fba4a4afcc083924d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 715.376081] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f0dba2afa07479fba4a4afcc083924d [ 715.709048] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.042s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 715.709684] env[64020]: ERROR nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4381921e-bc9e-4540-a293-d742eae29ad3, please check neutron logs for more information. [ 715.709684] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Traceback (most recent call last): [ 715.709684] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 715.709684] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] self.driver.spawn(context, instance, image_meta, [ 715.709684] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 715.709684] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 715.709684] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 715.709684] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] vm_ref = self.build_virtual_machine(instance, [ 715.709684] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 715.709684] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] vif_infos = vmwarevif.get_vif_info(self._session, [ 715.709684] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] for vif in network_info: [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] return self._sync_wrapper(fn, *args, **kwargs) [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] self.wait() [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] self[:] = self._gt.wait() [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] return self._exit_event.wait() [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] current.throw(*self._exc) [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 715.709989] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] result = function(*args, **kwargs) [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] return func(*args, **kwargs) [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] raise e [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] nwinfo = self.network_api.allocate_for_instance( [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] created_port_ids = self._update_ports_for_instance( [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] with excutils.save_and_reraise_exception(): [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] self.force_reraise() [ 715.710319] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 715.710780] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] raise self.value [ 715.710780] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 715.710780] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] updated_port = self._update_port( [ 715.710780] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 715.710780] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] _ensure_no_port_binding_failure(port) [ 715.710780] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 715.710780] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] raise exception.PortBindingFailed(port_id=port['id']) [ 715.710780] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] nova.exception.PortBindingFailed: Binding failed for port 4381921e-bc9e-4540-a293-d742eae29ad3, please check neutron logs for more information. [ 715.710780] env[64020]: ERROR nova.compute.manager [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] [ 715.710780] env[64020]: DEBUG nova.compute.utils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Binding failed for port 4381921e-bc9e-4540-a293-d742eae29ad3, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 715.711641] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.757s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 715.713190] env[64020]: INFO nova.compute.claims [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 715.714906] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 246a60524a214623bcdb411af48c57be in queue reply_57893177120949e6a93cb88e15cd42b4 [ 715.718767] env[64020]: DEBUG nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Build of instance 583b323c-a4e1-4431-9bc0-d9f4b7354cb6 was re-scheduled: Binding failed for port 4381921e-bc9e-4540-a293-d742eae29ad3, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 715.718767] env[64020]: DEBUG nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 715.718767] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Acquiring lock "refresh_cache-583b323c-a4e1-4431-9bc0-d9f4b7354cb6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 715.718767] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Acquired lock "refresh_cache-583b323c-a4e1-4431-9bc0-d9f4b7354cb6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 715.718990] env[64020]: DEBUG nova.network.neutron [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 715.718990] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 0b1ee007500541de900d31caf9bbc112 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 715.724657] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b1ee007500541de900d31caf9bbc112 [ 715.757669] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 246a60524a214623bcdb411af48c57be [ 715.857359] env[64020]: INFO nova.compute.manager [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: af8b2b66-c2d6-4bb6-a266-52cb82bac3be] Took 1.05 seconds to deallocate network for instance. [ 715.858391] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 08c7a4b23da749b686006956f0917cb9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 715.868313] env[64020]: DEBUG nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 715.870134] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 33b8c9c550ab4de5abfbb867e2289c97 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 715.901665] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08c7a4b23da749b686006956f0917cb9 [ 715.910150] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33b8c9c550ab4de5abfbb867e2289c97 [ 716.220564] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 93ddae6bda0049ec831ba30d9dd3caeb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 716.228608] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93ddae6bda0049ec831ba30d9dd3caeb [ 716.235444] env[64020]: DEBUG nova.network.neutron [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 716.346140] env[64020]: DEBUG nova.network.neutron [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 716.346910] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 73b6c51671fc4986a28bc5d5f6cfd0a3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 716.357427] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 73b6c51671fc4986a28bc5d5f6cfd0a3 [ 716.362437] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 2bdb4f8a930c47f5943c8b3a538a7706 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 716.392659] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 716.397158] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2bdb4f8a930c47f5943c8b3a538a7706 [ 716.852420] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Releasing lock "refresh_cache-583b323c-a4e1-4431-9bc0-d9f4b7354cb6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 716.852420] env[64020]: DEBUG nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 716.852420] env[64020]: DEBUG nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 716.852420] env[64020]: DEBUG nova.network.neutron [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 716.874175] env[64020]: DEBUG nova.network.neutron [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 716.874175] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 4c3d58b5ed5a43378009005a46c8b5a5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 716.879993] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c3d58b5ed5a43378009005a46c8b5a5 [ 716.894643] env[64020]: INFO nova.scheduler.client.report [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Deleted allocations for instance af8b2b66-c2d6-4bb6-a266-52cb82bac3be [ 716.902003] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg c28cb0cd46ab4ac7a8d4866fbd48c24f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 716.936885] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c28cb0cd46ab4ac7a8d4866fbd48c24f [ 717.154526] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-616d0f9b-0487-4234-8f3f-f51401650195 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.162290] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf032e60-9a68-410f-88cb-1fef6904a470 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.192488] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25913c36-3993-4651-9d8c-c770fda3b14e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.199731] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d49ce86-7f07-4820-b71e-ca98f004df3c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.213015] env[64020]: DEBUG nova.compute.provider_tree [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 717.213410] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 046985840bb549f7999b50dfc6aa60cb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 717.222353] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 046985840bb549f7999b50dfc6aa60cb [ 717.376288] env[64020]: DEBUG nova.network.neutron [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 717.376836] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg b1ae30d2e448497b849be1dff54e7bf8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 717.389933] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1ae30d2e448497b849be1dff54e7bf8 [ 717.406769] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f44f8253-e54a-4bda-bf2f-cc5618ddb236 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "af8b2b66-c2d6-4bb6-a266-52cb82bac3be" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 98.955s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 717.407304] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg b1c9cbad2d0142e49a010f3c91f6991d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 717.425259] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1c9cbad2d0142e49a010f3c91f6991d [ 717.559257] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquiring lock "8de44ad9-adb7-4589-b2d8-a2aeeb89892f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 717.559484] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "8de44ad9-adb7-4589-b2d8-a2aeeb89892f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 717.715766] env[64020]: DEBUG nova.scheduler.client.report [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 717.718224] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 9428a9d744694565b75d008eadc344bf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 717.729897] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9428a9d744694565b75d008eadc344bf [ 717.879897] env[64020]: INFO nova.compute.manager [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] [instance: 583b323c-a4e1-4431-9bc0-d9f4b7354cb6] Took 1.03 seconds to deallocate network for instance. [ 717.881821] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 7fc6ff962f01481a96abf9c65d2d8a1d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 717.909291] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 717.911043] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg bd9542bf1b9745a2ad6abd6861736c7f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 717.932676] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7fc6ff962f01481a96abf9c65d2d8a1d [ 717.950467] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bd9542bf1b9745a2ad6abd6861736c7f [ 718.227922] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.510s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 718.227922] env[64020]: DEBUG nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 718.227922] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 50efcc72039e40eaa0dfd995c23b1ba4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 718.227922] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.348s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 718.227922] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 6bfaa9a0e2a94e4c9cef3a837da3cf16 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 718.259308] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50efcc72039e40eaa0dfd995c23b1ba4 [ 718.262244] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6bfaa9a0e2a94e4c9cef3a837da3cf16 [ 718.386374] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg 13df964cce654c43bed578719006be4d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 718.425193] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 13df964cce654c43bed578719006be4d [ 718.433170] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 718.733071] env[64020]: DEBUG nova.compute.utils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 718.733672] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 9845bd91c57240a9934ac2a234a43b88 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 718.734525] env[64020]: DEBUG nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 718.734688] env[64020]: DEBUG nova.network.neutron [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 718.743890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9845bd91c57240a9934ac2a234a43b88 [ 718.842728] env[64020]: DEBUG nova.policy [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '58e498bf6e8148c8910bfbb9d67a7d0b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '464b10084e6b4306bf047cc0e6d593ab', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 718.911322] env[64020]: INFO nova.scheduler.client.report [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Deleted allocations for instance 583b323c-a4e1-4431-9bc0-d9f4b7354cb6 [ 718.936030] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Expecting reply to msg c449de1e039b4cd1a870e74e935ddb12 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 718.953000] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c449de1e039b4cd1a870e74e935ddb12 [ 719.228231] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af01acae-afbc-415c-bf1d-e57e8d58637f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.236231] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04e6a10a-c2d4-4094-9f7d-ac0afda5eda5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.239607] env[64020]: DEBUG nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 719.241728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg ebcdc38fcc5d498d9eaa81aec312d813 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 719.275222] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd8607bd-5d6c-44d0-afcd-398fcae6a88e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.283608] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ebcdc38fcc5d498d9eaa81aec312d813 [ 719.284628] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa4cf943-2ff1-4362-906a-fc0848b99778 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.302654] env[64020]: DEBUG nova.compute.provider_tree [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 719.303027] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 2d6b18e2a3ca4af093fbf821c755799b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 719.310072] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d6b18e2a3ca4af093fbf821c755799b [ 719.370754] env[64020]: DEBUG nova.network.neutron [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Successfully created port: 9e393b6b-5a62-425c-98a6-2cae8538407d {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 719.441350] env[64020]: DEBUG oslo_concurrency.lockutils [None req-81ab3170-b3e6-4d9a-a8a5-218b87121743 tempest-ServerAddressesNegativeTestJSON-1306227735 tempest-ServerAddressesNegativeTestJSON-1306227735-project-member] Lock "583b323c-a4e1-4431-9bc0-d9f4b7354cb6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 99.944s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 719.441994] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 06bfa5a0effe42dd9d3fed5aa9e99787 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 719.458107] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06bfa5a0effe42dd9d3fed5aa9e99787 [ 719.750755] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 374fbc2f854a4790a232a89a0cb30109 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 719.790889] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 374fbc2f854a4790a232a89a0cb30109 [ 719.805773] env[64020]: DEBUG nova.scheduler.client.report [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 719.808314] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 1d36410439f24415a4b1f75fec6bbb5c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 719.820995] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d36410439f24415a4b1f75fec6bbb5c [ 719.944415] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 719.946191] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 85ddd7acfad341538cf1e8fca0f241b0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 720.004010] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 85ddd7acfad341538cf1e8fca0f241b0 [ 720.253728] env[64020]: DEBUG nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 720.277935] env[64020]: DEBUG nova.virt.hardware [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 720.278177] env[64020]: DEBUG nova.virt.hardware [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 720.278324] env[64020]: DEBUG nova.virt.hardware [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 720.278499] env[64020]: DEBUG nova.virt.hardware [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 720.278640] env[64020]: DEBUG nova.virt.hardware [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 720.278780] env[64020]: DEBUG nova.virt.hardware [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 720.278984] env[64020]: DEBUG nova.virt.hardware [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 720.279134] env[64020]: DEBUG nova.virt.hardware [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 720.279295] env[64020]: DEBUG nova.virt.hardware [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 720.279452] env[64020]: DEBUG nova.virt.hardware [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 720.279612] env[64020]: DEBUG nova.virt.hardware [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 720.280495] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55ee8037-1fc8-4b4e-9a85-ba5eff4291c9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 720.291270] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8838866c-dce1-40f6-94d6-70b5fadcfbaa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 720.311791] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.086s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 720.312550] env[64020]: ERROR nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6d7c92ef-7904-4880-a4d2-89b79f6aaae6, please check neutron logs for more information. [ 720.312550] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Traceback (most recent call last): [ 720.312550] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 720.312550] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] self.driver.spawn(context, instance, image_meta, [ 720.312550] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 720.312550] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 720.312550] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 720.312550] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] vm_ref = self.build_virtual_machine(instance, [ 720.312550] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 720.312550] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] vif_infos = vmwarevif.get_vif_info(self._session, [ 720.312550] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] for vif in network_info: [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] return self._sync_wrapper(fn, *args, **kwargs) [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] self.wait() [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] self[:] = self._gt.wait() [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] return self._exit_event.wait() [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] current.throw(*self._exc) [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 720.312888] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] result = function(*args, **kwargs) [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] return func(*args, **kwargs) [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] raise e [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] nwinfo = self.network_api.allocate_for_instance( [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] created_port_ids = self._update_ports_for_instance( [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] with excutils.save_and_reraise_exception(): [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] self.force_reraise() [ 720.313194] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 720.313503] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] raise self.value [ 720.313503] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 720.313503] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] updated_port = self._update_port( [ 720.313503] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 720.313503] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] _ensure_no_port_binding_failure(port) [ 720.313503] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 720.313503] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] raise exception.PortBindingFailed(port_id=port['id']) [ 720.313503] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] nova.exception.PortBindingFailed: Binding failed for port 6d7c92ef-7904-4880-a4d2-89b79f6aaae6, please check neutron logs for more information. [ 720.313503] env[64020]: ERROR nova.compute.manager [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] [ 720.313503] env[64020]: DEBUG nova.compute.utils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Binding failed for port 6d7c92ef-7904-4880-a4d2-89b79f6aaae6, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 720.314493] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 17.388s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 720.315261] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg fae982880e674fd79330d77060ceb33a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 720.319638] env[64020]: DEBUG nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Build of instance 8368f535-f3d4-4b03-98ef-20b0bf91d25c was re-scheduled: Binding failed for port 6d7c92ef-7904-4880-a4d2-89b79f6aaae6, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 720.319638] env[64020]: DEBUG nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 720.319638] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "refresh_cache-8368f535-f3d4-4b03-98ef-20b0bf91d25c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 720.319638] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquired lock "refresh_cache-8368f535-f3d4-4b03-98ef-20b0bf91d25c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 720.319895] env[64020]: DEBUG nova.network.neutron [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 720.319895] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 1a8a9144caaa42d590bfe9cd4e688609 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 720.326010] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a8a9144caaa42d590bfe9cd4e688609 [ 720.339763] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fae982880e674fd79330d77060ceb33a [ 720.432270] env[64020]: DEBUG nova.compute.manager [req-e6aa83f5-28e8-40cf-8bb1-3083e33bc195 req-8099e746-25a1-4d0a-9d94-c3c5d3c77dc8 service nova] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Received event network-changed-9e393b6b-5a62-425c-98a6-2cae8538407d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 720.432468] env[64020]: DEBUG nova.compute.manager [req-e6aa83f5-28e8-40cf-8bb1-3083e33bc195 req-8099e746-25a1-4d0a-9d94-c3c5d3c77dc8 service nova] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Refreshing instance network info cache due to event network-changed-9e393b6b-5a62-425c-98a6-2cae8538407d. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 720.432675] env[64020]: DEBUG oslo_concurrency.lockutils [req-e6aa83f5-28e8-40cf-8bb1-3083e33bc195 req-8099e746-25a1-4d0a-9d94-c3c5d3c77dc8 service nova] Acquiring lock "refresh_cache-5e43d3ea-740c-4569-be82-3bb7082fa8ea" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 720.432811] env[64020]: DEBUG oslo_concurrency.lockutils [req-e6aa83f5-28e8-40cf-8bb1-3083e33bc195 req-8099e746-25a1-4d0a-9d94-c3c5d3c77dc8 service nova] Acquired lock "refresh_cache-5e43d3ea-740c-4569-be82-3bb7082fa8ea" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 720.432964] env[64020]: DEBUG nova.network.neutron [req-e6aa83f5-28e8-40cf-8bb1-3083e33bc195 req-8099e746-25a1-4d0a-9d94-c3c5d3c77dc8 service nova] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Refreshing network info cache for port 9e393b6b-5a62-425c-98a6-2cae8538407d {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 720.433373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e6aa83f5-28e8-40cf-8bb1-3083e33bc195 req-8099e746-25a1-4d0a-9d94-c3c5d3c77dc8 service nova] Expecting reply to msg 2ec66c3c57f24f7484af2bbb33c43213 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 720.439527] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ec66c3c57f24f7484af2bbb33c43213 [ 720.467043] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 720.654917] env[64020]: ERROR nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9e393b6b-5a62-425c-98a6-2cae8538407d, please check neutron logs for more information. [ 720.654917] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 720.654917] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 720.654917] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 720.654917] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 720.654917] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 720.654917] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 720.654917] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 720.654917] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 720.654917] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 720.654917] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 720.654917] env[64020]: ERROR nova.compute.manager raise self.value [ 720.654917] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 720.654917] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 720.654917] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 720.654917] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 720.655406] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 720.655406] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 720.655406] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9e393b6b-5a62-425c-98a6-2cae8538407d, please check neutron logs for more information. [ 720.655406] env[64020]: ERROR nova.compute.manager [ 720.655406] env[64020]: Traceback (most recent call last): [ 720.655406] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 720.655406] env[64020]: listener.cb(fileno) [ 720.655406] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 720.655406] env[64020]: result = function(*args, **kwargs) [ 720.655406] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 720.655406] env[64020]: return func(*args, **kwargs) [ 720.655406] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 720.655406] env[64020]: raise e [ 720.655406] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 720.655406] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 720.655406] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 720.655406] env[64020]: created_port_ids = self._update_ports_for_instance( [ 720.655406] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 720.655406] env[64020]: with excutils.save_and_reraise_exception(): [ 720.655406] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 720.655406] env[64020]: self.force_reraise() [ 720.655406] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 720.655406] env[64020]: raise self.value [ 720.655406] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 720.655406] env[64020]: updated_port = self._update_port( [ 720.655406] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 720.655406] env[64020]: _ensure_no_port_binding_failure(port) [ 720.655406] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 720.655406] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 720.656122] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 9e393b6b-5a62-425c-98a6-2cae8538407d, please check neutron logs for more information. [ 720.656122] env[64020]: Removing descriptor: 16 [ 720.656122] env[64020]: ERROR nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9e393b6b-5a62-425c-98a6-2cae8538407d, please check neutron logs for more information. [ 720.656122] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Traceback (most recent call last): [ 720.656122] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 720.656122] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] yield resources [ 720.656122] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 720.656122] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] self.driver.spawn(context, instance, image_meta, [ 720.656122] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 720.656122] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] self._vmops.spawn(context, instance, image_meta, injected_files, [ 720.656122] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 720.656122] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] vm_ref = self.build_virtual_machine(instance, [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] vif_infos = vmwarevif.get_vif_info(self._session, [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] for vif in network_info: [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] return self._sync_wrapper(fn, *args, **kwargs) [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] self.wait() [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] self[:] = self._gt.wait() [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] return self._exit_event.wait() [ 720.656450] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] result = hub.switch() [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] return self.greenlet.switch() [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] result = function(*args, **kwargs) [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] return func(*args, **kwargs) [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] raise e [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] nwinfo = self.network_api.allocate_for_instance( [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 720.656761] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] created_port_ids = self._update_ports_for_instance( [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] with excutils.save_and_reraise_exception(): [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] self.force_reraise() [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] raise self.value [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] updated_port = self._update_port( [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] _ensure_no_port_binding_failure(port) [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 720.657071] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] raise exception.PortBindingFailed(port_id=port['id']) [ 720.657363] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] nova.exception.PortBindingFailed: Binding failed for port 9e393b6b-5a62-425c-98a6-2cae8538407d, please check neutron logs for more information. [ 720.657363] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] [ 720.657363] env[64020]: INFO nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Terminating instance [ 720.657850] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Acquiring lock "refresh_cache-5e43d3ea-740c-4569-be82-3bb7082fa8ea" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 720.821406] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg f96fb4dd2503428e9a0c80930085cb3f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 720.837675] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f96fb4dd2503428e9a0c80930085cb3f [ 720.842223] env[64020]: DEBUG nova.network.neutron [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 720.962155] env[64020]: DEBUG nova.network.neutron [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 720.962670] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 819f43ca5a094deea6c532372b7fd83f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 720.973009] env[64020]: DEBUG nova.network.neutron [req-e6aa83f5-28e8-40cf-8bb1-3083e33bc195 req-8099e746-25a1-4d0a-9d94-c3c5d3c77dc8 service nova] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 720.975265] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 819f43ca5a094deea6c532372b7fd83f [ 721.106503] env[64020]: DEBUG nova.network.neutron [req-e6aa83f5-28e8-40cf-8bb1-3083e33bc195 req-8099e746-25a1-4d0a-9d94-c3c5d3c77dc8 service nova] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 721.107061] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e6aa83f5-28e8-40cf-8bb1-3083e33bc195 req-8099e746-25a1-4d0a-9d94-c3c5d3c77dc8 service nova] Expecting reply to msg b3ca8dcb08544e22b8f6de89f18c5b10 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 721.115389] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3ca8dcb08544e22b8f6de89f18c5b10 [ 721.350592] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg c619d12f646f43c4bd2cc7c76b3ff869 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 721.362479] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c619d12f646f43c4bd2cc7c76b3ff869 [ 721.466925] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Releasing lock "refresh_cache-8368f535-f3d4-4b03-98ef-20b0bf91d25c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 721.466925] env[64020]: DEBUG nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 721.466925] env[64020]: DEBUG nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 721.466925] env[64020]: DEBUG nova.network.neutron [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 721.480823] env[64020]: DEBUG nova.network.neutron [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 721.481433] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 0efaae0a69e146fdabc2a6ca4e0f4d61 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 721.489524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0efaae0a69e146fdabc2a6ca4e0f4d61 [ 721.609300] env[64020]: DEBUG oslo_concurrency.lockutils [req-e6aa83f5-28e8-40cf-8bb1-3083e33bc195 req-8099e746-25a1-4d0a-9d94-c3c5d3c77dc8 service nova] Releasing lock "refresh_cache-5e43d3ea-740c-4569-be82-3bb7082fa8ea" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 721.609738] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Acquired lock "refresh_cache-5e43d3ea-740c-4569-be82-3bb7082fa8ea" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 721.609937] env[64020]: DEBUG nova.network.neutron [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 721.610443] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 4b7c092f6c424a7ab92c5786f4b88298 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 721.619254] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4b7c092f6c424a7ab92c5786f4b88298 [ 721.853283] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 8368f535-f3d4-4b03-98ef-20b0bf91d25c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 721.853457] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 979aa8d0-1e9b-4bbb-b507-31f72542d8d0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 256, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 721.853582] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 0d97356d-3abf-4207-842d-6e2d468ef97c actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 721.853700] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 727e7787-68b7-441c-a31d-6f0796c33be0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 721.854262] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg edcfa3688d1a438c90a68f99104136a9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 721.864576] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg edcfa3688d1a438c90a68f99104136a9 [ 721.983423] env[64020]: DEBUG nova.network.neutron [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 721.984099] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg e599d82ae7854c9086698fd38dae0a77 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 721.992552] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e599d82ae7854c9086698fd38dae0a77 [ 722.126575] env[64020]: DEBUG nova.network.neutron [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 722.213451] env[64020]: DEBUG nova.network.neutron [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 722.213451] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg c462383a4fde4c4fb153afd920c2da6e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 722.214882] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c462383a4fde4c4fb153afd920c2da6e [ 722.356381] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 866f8ba4-faad-4883-80ca-4c4dedb2b50c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 722.356541] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 5e43d3ea-740c-4569-be82-3bb7082fa8ea actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 722.357088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 296355c9be974d8eb593fa599c1aa1a6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 722.368407] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 296355c9be974d8eb593fa599c1aa1a6 [ 722.457442] env[64020]: DEBUG nova.compute.manager [req-bbf8b7f2-e9d3-4b59-b6c5-03fb958012f8 req-9add3489-b08e-4b78-b841-01384e972fdf service nova] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Received event network-vif-deleted-9e393b6b-5a62-425c-98a6-2cae8538407d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 722.486747] env[64020]: INFO nova.compute.manager [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 8368f535-f3d4-4b03-98ef-20b0bf91d25c] Took 1.02 seconds to deallocate network for instance. [ 722.488441] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 37bbd997a086421fa60f7d13d05e2759 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 722.525034] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37bbd997a086421fa60f7d13d05e2759 [ 722.709485] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Releasing lock "refresh_cache-5e43d3ea-740c-4569-be82-3bb7082fa8ea" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 722.709990] env[64020]: DEBUG nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 722.710185] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 722.710504] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4f45f86b-30d2-4c6e-a5e6-f285fdd8008e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 722.720325] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da7ea32f-9ce4-445b-ae2d-74afc0b0bc60 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 722.746829] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5e43d3ea-740c-4569-be82-3bb7082fa8ea could not be found. [ 722.747070] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 722.747245] env[64020]: INFO nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Took 0.04 seconds to destroy the instance on the hypervisor. [ 722.747932] env[64020]: DEBUG oslo.service.loopingcall [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 722.748196] env[64020]: DEBUG nova.compute.manager [-] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 722.748294] env[64020]: DEBUG nova.network.neutron [-] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 722.768872] env[64020]: DEBUG nova.network.neutron [-] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 722.769386] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 99fa5094ad9744d6b8ef9c41de9c9400 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 722.775993] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99fa5094ad9744d6b8ef9c41de9c9400 [ 722.860116] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 371d5fb1-f1b6-41de-a4e6-88e8137f4be9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 722.860717] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 7678586e579142a18ebd2052c0acd440 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 722.871634] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7678586e579142a18ebd2052c0acd440 [ 722.992579] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg d23012eb01bf4c0e81c6303f3825c5a4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 723.023437] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d23012eb01bf4c0e81c6303f3825c5a4 [ 723.271614] env[64020]: DEBUG nova.network.neutron [-] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 723.272157] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c169f59c1e4f4e73854f28fd4283c842 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 723.284476] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c169f59c1e4f4e73854f28fd4283c842 [ 723.363314] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance a26baa75-d36d-4c05-bde8-d39fe814191d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 723.363979] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 01de61531cb34ef9a53492ac930cf962 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 723.376147] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 01de61531cb34ef9a53492ac930cf962 [ 723.521057] env[64020]: INFO nova.scheduler.client.report [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Deleted allocations for instance 8368f535-f3d4-4b03-98ef-20b0bf91d25c [ 723.532620] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 7bcd7606240447c4804dbb71391e69e7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 723.556024] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7bcd7606240447c4804dbb71391e69e7 [ 723.776115] env[64020]: INFO nova.compute.manager [-] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Took 1.03 seconds to deallocate network for instance. [ 723.778608] env[64020]: DEBUG nova.compute.claims [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 723.779956] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 723.867323] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance e01583c9-744c-47ea-b7cf-a46b046fc854 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 723.867926] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 37a82ed954ea4f1d947b76c7f7eee870 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 723.878689] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37a82ed954ea4f1d947b76c7f7eee870 [ 724.040100] env[64020]: DEBUG oslo_concurrency.lockutils [None req-86f4991e-1a9b-407e-8403-d6a99050386a tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "8368f535-f3d4-4b03-98ef-20b0bf91d25c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 103.124s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 724.040721] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 1cdf008d8a0e4b359050b7c6113ee5eb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 724.052686] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1cdf008d8a0e4b359050b7c6113ee5eb [ 724.370053] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 786a572c-adaa-4656-8b42-c510940dac59 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 724.371476] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 9b070f1d682940ad8af27cf5503281e5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 724.383044] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b070f1d682940ad8af27cf5503281e5 [ 724.545787] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 724.545787] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 877c9993a4c24e5cb54884f6e72ee017 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 724.579550] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 877c9993a4c24e5cb54884f6e72ee017 [ 724.875006] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 8e8e1b50-fcae-41c6-8f5a-241692140cda has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 724.875006] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 7191b37c24bd46cd870c31ba430fdede in queue reply_57893177120949e6a93cb88e15cd42b4 [ 724.885379] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7191b37c24bd46cd870c31ba430fdede [ 725.069713] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 725.378752] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance d808804b-a2d0-4225-beb5-7d91e0cff3ac has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 725.378752] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 20effee243be4fae8f4f0800da71d821 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 725.389606] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20effee243be4fae8f4f0800da71d821 [ 725.826810] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquiring lock "f7f8d239-aff8-4cd7-afdd-974c53b3e563" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 725.826810] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "f7f8d239-aff8-4cd7-afdd-974c53b3e563" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 725.880669] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 725.881510] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 6527aecdf4ef4960b5b4d029d042e462 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 725.894024] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6527aecdf4ef4960b5b4d029d042e462 [ 726.384527] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 47516f85-60d8-4931-a205-b2ec1f7aec83 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 726.385109] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg b4c9d2be80c142569ed11997e809afa8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 726.397393] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b4c9d2be80c142569ed11997e809afa8 [ 726.888062] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 726.888659] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 2e6e6b2e13f04a2f8ce9e94508683e4b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 726.899008] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2e6e6b2e13f04a2f8ce9e94508683e4b [ 727.391146] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance f873c026-0fa1-4176-a30f-207767f87410 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 727.391727] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 92a068ca337549a8ac3e7531ad856a1c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 727.402792] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92a068ca337549a8ac3e7531ad856a1c [ 727.894226] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 052ae9f0-bb03-4747-b8f9-708e4a7c147c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 727.894798] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg a70165c469e148cf859ac10f17f04252 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 727.905087] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a70165c469e148cf859ac10f17f04252 [ 728.397861] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance b89f34cd-bd7b-4f22-8e6f-a167d9f01af8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 728.399372] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 1bfa2cc7be2645a6ab5109b461e526fd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 728.409654] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1bfa2cc7be2645a6ab5109b461e526fd [ 728.901435] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 1954e745-783e-417e-aafc-265da55066cf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 728.902012] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg a0255c4c9f7e4fe4a61cb12ba0267d90 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 728.911973] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a0255c4c9f7e4fe4a61cb12ba0267d90 [ 729.404638] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 5ccefb28-25a8-4876-a31c-58c7edfc9570 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 729.405300] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 0fd9dc1099c04892a0b6e7fb1fad5cab in queue reply_57893177120949e6a93cb88e15cd42b4 [ 729.417091] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0fd9dc1099c04892a0b6e7fb1fad5cab [ 729.907824] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance a39b40c8-0340-4778-8b43-f29ed51664bc has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 729.908444] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg fa7dc75fcd7749c585a7e8564e33e059 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 729.918873] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fa7dc75fcd7749c585a7e8564e33e059 [ 730.411279] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 730.411879] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 2096ea8a0dcb4cdd99e6b0f24e4a642b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 730.422989] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2096ea8a0dcb4cdd99e6b0f24e4a642b [ 730.914458] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance d6228174-ceb1-4ca0-b62b-27db15c20a85 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 730.915026] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 1a6af74186754caf845f4a35ca575301 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 730.925688] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a6af74186754caf845f4a35ca575301 [ 731.417591] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance e27f988a-a64b-407b-ad66-4bed7ee45b52 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 731.418170] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 572afa61789d4ab3be6ca0c85ab0d43c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 731.429833] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 572afa61789d4ab3be6ca0c85ab0d43c [ 731.920865] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 731.921705] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 9e05d940245049eda859499a7ff444e1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 731.931768] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e05d940245049eda859499a7ff444e1 [ 732.425068] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 995288b5-d79a-4af3-a1e8-3571fff2d356 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 732.425643] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 2f55c5a8de40470d8fc1ab7fa2ff0d25 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 732.436201] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f55c5a8de40470d8fc1ab7fa2ff0d25 [ 732.927997] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 45167852-b7c0-4614-89f1-f8f7fc2078f2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 732.928594] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 708045912369497b89e5bf6800d69062 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 732.938970] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 708045912369497b89e5bf6800d69062 [ 733.431472] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance d567d3dd-5820-4d6c-8922-3e39a1d9ebd6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 733.432055] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg a8a9d1ccbeae46a0aa4f0854a94a3f75 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 733.443038] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8a9d1ccbeae46a0aa4f0854a94a3f75 [ 733.938108] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance ecb7f281-1206-4693-845b-c59f639b0789 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 733.938108] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg f28ff3105cf74a5ab9038bbcc01ade61 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 733.948510] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f28ff3105cf74a5ab9038bbcc01ade61 [ 734.442948] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 8de44ad9-adb7-4589-b2d8-a2aeeb89892f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 734.442948] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Total usable vcpus: 48, total allocated vcpus: 4 {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 734.442948] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1344MB phys_disk=200GB used_disk=4GB total_vcpus=48 used_vcpus=4 pci_stats=[] {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 734.811751] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24935efb-f05d-49e6-b33d-fb54e8bd7794 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.820320] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b45cf85e-f7d2-4e97-bf33-41afb78528b2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.848972] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58f72c0e-25ea-4c24-ba7e-043a8cffa77d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.857357] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26a14ccb-63a2-435d-b8b2-c1210486cf27 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.870536] env[64020]: DEBUG nova.compute.provider_tree [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 734.871006] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg ec8e09e0eca049c7b180011c34ca400d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 734.878498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec8e09e0eca049c7b180011c34ca400d [ 735.375889] env[64020]: DEBUG nova.scheduler.client.report [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 735.377247] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 3f33391902cb4c5f8a77a2ec7b0959d6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 735.392627] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f33391902cb4c5f8a77a2ec7b0959d6 [ 735.879927] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=64020) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 735.880218] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 15.566s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.880462] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 30.952s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.882362] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 5e59d9ac6ac0403fb6077b66b69b6614 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 735.920664] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e59d9ac6ac0403fb6077b66b69b6614 [ 736.686087] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0eab557d-9760-43d8-9698-28a8606aa3ac {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.693528] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ceafe334-b3f0-4794-bba3-a44ffabef10d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.722764] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ce0f9a7-0144-45ec-b703-2470da3292fb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.729507] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b835adfc-e91b-491d-9fe2-46c1120650a8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.741917] env[64020]: DEBUG nova.compute.provider_tree [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 736.742405] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 8f8792e1559542edb001cb2790b50a1d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 736.749805] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f8792e1559542edb001cb2790b50a1d [ 737.245541] env[64020]: DEBUG nova.scheduler.client.report [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 737.247918] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 1e48c608eb3b4ebea5f91eb6b40fd750 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 737.258634] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e48c608eb3b4ebea5f91eb6b40fd750 [ 737.750638] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.870s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 737.751313] env[64020]: ERROR nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 89cee47c-37ae-4a0d-8184-0c7c53ff7bb1, please check neutron logs for more information. [ 737.751313] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Traceback (most recent call last): [ 737.751313] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 737.751313] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] self.driver.spawn(context, instance, image_meta, [ 737.751313] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 737.751313] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 737.751313] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 737.751313] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] vm_ref = self.build_virtual_machine(instance, [ 737.751313] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 737.751313] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] vif_infos = vmwarevif.get_vif_info(self._session, [ 737.751313] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] for vif in network_info: [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] return self._sync_wrapper(fn, *args, **kwargs) [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] self.wait() [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] self[:] = self._gt.wait() [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] return self._exit_event.wait() [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] result = hub.switch() [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 737.751678] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] return self.greenlet.switch() [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] result = function(*args, **kwargs) [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] return func(*args, **kwargs) [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] raise e [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] nwinfo = self.network_api.allocate_for_instance( [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] created_port_ids = self._update_ports_for_instance( [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] with excutils.save_and_reraise_exception(): [ 737.752164] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 737.752560] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] self.force_reraise() [ 737.752560] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 737.752560] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] raise self.value [ 737.752560] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 737.752560] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] updated_port = self._update_port( [ 737.752560] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 737.752560] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] _ensure_no_port_binding_failure(port) [ 737.752560] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 737.752560] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] raise exception.PortBindingFailed(port_id=port['id']) [ 737.752560] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] nova.exception.PortBindingFailed: Binding failed for port 89cee47c-37ae-4a0d-8184-0c7c53ff7bb1, please check neutron logs for more information. [ 737.752560] env[64020]: ERROR nova.compute.manager [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] [ 737.752886] env[64020]: DEBUG nova.compute.utils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Binding failed for port 89cee47c-37ae-4a0d-8184-0c7c53ff7bb1, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 737.753458] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 30.579s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 737.755338] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg d974553fdf3942d0876a4b4e62ba5651 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 737.756544] env[64020]: DEBUG nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Build of instance 979aa8d0-1e9b-4bbb-b507-31f72542d8d0 was re-scheduled: Binding failed for port 89cee47c-37ae-4a0d-8184-0c7c53ff7bb1, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 737.756952] env[64020]: DEBUG nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 737.757175] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquiring lock "refresh_cache-979aa8d0-1e9b-4bbb-b507-31f72542d8d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 737.757322] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Acquired lock "refresh_cache-979aa8d0-1e9b-4bbb-b507-31f72542d8d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 737.757481] env[64020]: DEBUG nova.network.neutron [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 737.757848] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 43fcdf19d1f3458093858702338d0184 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 737.764496] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43fcdf19d1f3458093858702338d0184 [ 737.785901] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d974553fdf3942d0876a4b4e62ba5651 [ 738.278037] env[64020]: DEBUG nova.network.neutron [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 738.360602] env[64020]: DEBUG nova.network.neutron [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 738.361113] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 820b7b755cfa438bbdd3d8fe8b690cf4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 738.370090] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 820b7b755cfa438bbdd3d8fe8b690cf4 [ 738.593684] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-beb73750-0866-4eac-b394-b7e42df3322c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 738.601611] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a811c66-f8fd-45ed-b7ff-9ae68004f170 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 738.634269] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d40fb35-b5c3-4019-8d73-58cafca1baa9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 738.641898] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5db4f999-597e-456f-8754-2a1b46246693 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 738.655052] env[64020]: DEBUG nova.compute.provider_tree [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 738.655594] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg c74c8569e4d1428fa8d3acad91b124e4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 738.662683] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c74c8569e4d1428fa8d3acad91b124e4 [ 738.864468] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Releasing lock "refresh_cache-979aa8d0-1e9b-4bbb-b507-31f72542d8d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 738.864743] env[64020]: DEBUG nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 738.864929] env[64020]: DEBUG nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 738.865097] env[64020]: DEBUG nova.network.neutron [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 738.887234] env[64020]: DEBUG nova.network.neutron [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 738.887234] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 1fda05bee6a04345aa717eb406222c69 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 738.901036] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1fda05bee6a04345aa717eb406222c69 [ 739.158183] env[64020]: DEBUG nova.scheduler.client.report [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 739.160771] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 564acc8454534839ad48c5b320085c4c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 739.172077] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 564acc8454534839ad48c5b320085c4c [ 739.398706] env[64020]: DEBUG nova.network.neutron [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 739.398706] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 8616aed88726444ab9873f9aae270e08 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 739.399254] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8616aed88726444ab9873f9aae270e08 [ 739.664173] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.910s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 739.664173] env[64020]: ERROR nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port aa2cb781-03b3-4ff9-a6ae-989b384497fb, please check neutron logs for more information. [ 739.664173] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Traceback (most recent call last): [ 739.664173] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 739.664173] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] self.driver.spawn(context, instance, image_meta, [ 739.664173] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 739.664173] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 739.664173] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 739.664173] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] vm_ref = self.build_virtual_machine(instance, [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] vif_infos = vmwarevif.get_vif_info(self._session, [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] for vif in network_info: [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] return self._sync_wrapper(fn, *args, **kwargs) [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] self.wait() [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] self[:] = self._gt.wait() [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] return self._exit_event.wait() [ 739.664858] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] current.throw(*self._exc) [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] result = function(*args, **kwargs) [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] return func(*args, **kwargs) [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] raise e [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] nwinfo = self.network_api.allocate_for_instance( [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] created_port_ids = self._update_ports_for_instance( [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 739.665334] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] with excutils.save_and_reraise_exception(): [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] self.force_reraise() [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] raise self.value [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] updated_port = self._update_port( [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] _ensure_no_port_binding_failure(port) [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] raise exception.PortBindingFailed(port_id=port['id']) [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] nova.exception.PortBindingFailed: Binding failed for port aa2cb781-03b3-4ff9-a6ae-989b384497fb, please check neutron logs for more information. [ 739.665725] env[64020]: ERROR nova.compute.manager [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] [ 739.666094] env[64020]: DEBUG nova.compute.utils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Binding failed for port aa2cb781-03b3-4ff9-a6ae-989b384497fb, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 739.666094] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 29.819s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 739.667728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 5b77db7fbedd42c7a71593d9d045385b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 739.669043] env[64020]: DEBUG nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Build of instance 0d97356d-3abf-4207-842d-6e2d468ef97c was re-scheduled: Binding failed for port aa2cb781-03b3-4ff9-a6ae-989b384497fb, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 739.679194] env[64020]: DEBUG nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 739.679455] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "refresh_cache-0d97356d-3abf-4207-842d-6e2d468ef97c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 739.679625] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquired lock "refresh_cache-0d97356d-3abf-4207-842d-6e2d468ef97c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 739.679791] env[64020]: DEBUG nova.network.neutron [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 739.680198] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 7a72ac00b31343fba3a0728aba008bc0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 739.686797] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a72ac00b31343fba3a0728aba008bc0 [ 739.709242] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b77db7fbedd42c7a71593d9d045385b [ 739.891933] env[64020]: INFO nova.compute.manager [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] [instance: 979aa8d0-1e9b-4bbb-b507-31f72542d8d0] Took 1.03 seconds to deallocate network for instance. [ 739.893805] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg cab75de12dc544ffbf74b8742660931e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 739.925075] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cab75de12dc544ffbf74b8742660931e [ 740.200387] env[64020]: DEBUG nova.network.neutron [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 740.311914] env[64020]: DEBUG nova.network.neutron [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 740.312468] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg c419990807c44838a3174db70e984450 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 740.324868] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c419990807c44838a3174db70e984450 [ 740.397632] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg bb717c0b25e64146a3f08626286cde2b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 740.440822] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb717c0b25e64146a3f08626286cde2b [ 740.580775] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd9446dd-965c-488f-be45-ce001ddf5c65 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.588335] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4589664e-627c-48d8-825a-7e21c9ebc98c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.627666] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3aacec02-0eba-4277-bce3-85a5110b2656 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.635047] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87a2eeda-2c93-4fca-b49d-050c61edd0a6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.648020] env[64020]: DEBUG nova.compute.provider_tree [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 740.648607] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 9b9c3197e9e5471ea269ceccd15de38e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 740.655983] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b9c3197e9e5471ea269ceccd15de38e [ 740.817831] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Releasing lock "refresh_cache-0d97356d-3abf-4207-842d-6e2d468ef97c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 740.818074] env[64020]: DEBUG nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 740.818291] env[64020]: DEBUG nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 740.818429] env[64020]: DEBUG nova.network.neutron [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 740.832793] env[64020]: DEBUG nova.network.neutron [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 740.833318] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 5707c9bd8cbb4acdbc43d2eca3d3cd19 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 740.839779] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5707c9bd8cbb4acdbc43d2eca3d3cd19 [ 740.918614] env[64020]: INFO nova.scheduler.client.report [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Deleted allocations for instance 979aa8d0-1e9b-4bbb-b507-31f72542d8d0 [ 740.924575] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Expecting reply to msg 3cb6481e1aa549ba98eb38bddd3d4d22 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 740.941566] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3cb6481e1aa549ba98eb38bddd3d4d22 [ 741.151376] env[64020]: DEBUG nova.scheduler.client.report [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 741.153849] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg f9b6fd73b0df4d3aa700e6ffdeba40e0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 741.164664] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f9b6fd73b0df4d3aa700e6ffdeba40e0 [ 741.335297] env[64020]: DEBUG nova.network.neutron [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 741.335820] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 5de1f666dfa64611985315dae93194b4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 741.344290] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5de1f666dfa64611985315dae93194b4 [ 741.430045] env[64020]: DEBUG oslo_concurrency.lockutils [None req-677d83ab-7b70-48b1-8636-dea43276fdf9 tempest-ListServerFiltersTestJSON-267105360 tempest-ListServerFiltersTestJSON-267105360-project-member] Lock "979aa8d0-1e9b-4bbb-b507-31f72542d8d0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 118.719s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 741.430624] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg a12d120e9c774937ac958fc2f64a7a85 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 741.442325] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a12d120e9c774937ac958fc2f64a7a85 [ 741.656983] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.991s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 741.657585] env[64020]: ERROR nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4, please check neutron logs for more information. [ 741.657585] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Traceback (most recent call last): [ 741.657585] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 741.657585] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] self.driver.spawn(context, instance, image_meta, [ 741.657585] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 741.657585] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 741.657585] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 741.657585] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] vm_ref = self.build_virtual_machine(instance, [ 741.657585] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 741.657585] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] vif_infos = vmwarevif.get_vif_info(self._session, [ 741.657585] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] for vif in network_info: [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] return self._sync_wrapper(fn, *args, **kwargs) [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] self.wait() [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] self[:] = self._gt.wait() [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] return self._exit_event.wait() [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] current.throw(*self._exc) [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 741.657918] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] result = function(*args, **kwargs) [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] return func(*args, **kwargs) [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] raise e [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] nwinfo = self.network_api.allocate_for_instance( [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] created_port_ids = self._update_ports_for_instance( [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] with excutils.save_and_reraise_exception(): [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] self.force_reraise() [ 741.658270] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 741.658617] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] raise self.value [ 741.658617] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 741.658617] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] updated_port = self._update_port( [ 741.658617] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 741.658617] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] _ensure_no_port_binding_failure(port) [ 741.658617] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 741.658617] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] raise exception.PortBindingFailed(port_id=port['id']) [ 741.658617] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] nova.exception.PortBindingFailed: Binding failed for port 00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4, please check neutron logs for more information. [ 741.658617] env[64020]: ERROR nova.compute.manager [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] [ 741.658617] env[64020]: DEBUG nova.compute.utils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Binding failed for port 00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 741.659596] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.258s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 741.661017] env[64020]: INFO nova.compute.claims [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 741.662613] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 7a210573027c46f0a689faf815bc6a93 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 741.664535] env[64020]: DEBUG nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Build of instance 727e7787-68b7-441c-a31d-6f0796c33be0 was re-scheduled: Binding failed for port 00b2e9cf-44a0-4a4a-bda9-f7df3e6028a4, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 741.665066] env[64020]: DEBUG nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 741.665285] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Acquiring lock "refresh_cache-727e7787-68b7-441c-a31d-6f0796c33be0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 741.665425] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Acquired lock "refresh_cache-727e7787-68b7-441c-a31d-6f0796c33be0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 741.665576] env[64020]: DEBUG nova.network.neutron [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 741.665927] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 44817bcd376846dfa0fe84df5ee00789 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 741.673739] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44817bcd376846dfa0fe84df5ee00789 [ 741.696130] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a210573027c46f0a689faf815bc6a93 [ 741.837917] env[64020]: INFO nova.compute.manager [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: 0d97356d-3abf-4207-842d-6e2d468ef97c] Took 1.02 seconds to deallocate network for instance. [ 741.839645] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg ff567333453a4214839139263bb995c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 741.874511] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff567333453a4214839139263bb995c8 [ 741.932488] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 741.934314] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 744786e47b0f4809a7fc5b5c502514a6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 741.966535] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 744786e47b0f4809a7fc5b5c502514a6 [ 742.167026] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 77eb7803d7264fb589ab8c3556573df0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 742.174824] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 77eb7803d7264fb589ab8c3556573df0 [ 742.197933] env[64020]: DEBUG nova.network.neutron [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 742.314517] env[64020]: DEBUG nova.network.neutron [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 742.315041] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 72ae51367ce340098587188262041543 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 742.323917] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72ae51367ce340098587188262041543 [ 742.344141] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg d80ffabe027a4952bb9b7dfdcb98ec7a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 742.376192] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d80ffabe027a4952bb9b7dfdcb98ec7a [ 742.453956] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 742.818380] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Releasing lock "refresh_cache-727e7787-68b7-441c-a31d-6f0796c33be0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 742.818726] env[64020]: DEBUG nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 742.819180] env[64020]: DEBUG nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 742.819418] env[64020]: DEBUG nova.network.neutron [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 742.836731] env[64020]: DEBUG nova.network.neutron [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 742.837542] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg bea5acee9d44467483ba35423ee30cb4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 742.845186] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bea5acee9d44467483ba35423ee30cb4 [ 742.866595] env[64020]: INFO nova.scheduler.client.report [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Deleted allocations for instance 0d97356d-3abf-4207-842d-6e2d468ef97c [ 742.873114] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 0258f90ac658460a9653c84ff0b76a3d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 742.889420] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0258f90ac658460a9653c84ff0b76a3d [ 743.062917] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf0cae00-f368-4f18-9556-cc1bb248d79a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.072897] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d625999-55c4-4448-aff3-913d5eb2754b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.121214] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4601fc53-66cd-423c-b907-921ad08e83f6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.130938] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-438db3b5-1c1b-4108-bd49-0bdcf193363b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.149374] env[64020]: DEBUG nova.compute.provider_tree [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 743.150042] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 2fce9d65f25d4fddaf81eb5ac4fa6fdf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 743.157351] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2fce9d65f25d4fddaf81eb5ac4fa6fdf [ 743.346127] env[64020]: DEBUG nova.network.neutron [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 743.346661] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 8e565219a11f445e8152a701afbbd219 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 743.355035] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8e565219a11f445e8152a701afbbd219 [ 743.378562] env[64020]: DEBUG oslo_concurrency.lockutils [None req-bd3445c5-3395-46cd-8f15-73adce80e30d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "0d97356d-3abf-4207-842d-6e2d468ef97c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 120.343s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 743.379085] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 98134001c50d4c83b8a5a910f9a05332 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 743.394051] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98134001c50d4c83b8a5a910f9a05332 [ 743.652758] env[64020]: DEBUG nova.scheduler.client.report [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 743.655233] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 20c6829922004e2683dfb758cd35c794 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 743.665599] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20c6829922004e2683dfb758cd35c794 [ 743.849053] env[64020]: INFO nova.compute.manager [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] [instance: 727e7787-68b7-441c-a31d-6f0796c33be0] Took 1.03 seconds to deallocate network for instance. [ 743.851146] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg bdea8f42f7b74e39b36cdac41ccd9043 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 743.881427] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 743.883233] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg be30b5944bf04a479716f42fba1b2dca in queue reply_57893177120949e6a93cb88e15cd42b4 [ 743.886706] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bdea8f42f7b74e39b36cdac41ccd9043 [ 743.911667] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be30b5944bf04a479716f42fba1b2dca [ 744.157969] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.498s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.158489] env[64020]: DEBUG nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 744.160119] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg fbd25858e8f44abeb002f45a787282a5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 744.161143] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.800s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.162511] env[64020]: INFO nova.compute.claims [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 744.163970] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg 2f0a61cb1a7944689734ee4c5f467858 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 744.192935] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fbd25858e8f44abeb002f45a787282a5 [ 744.199345] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f0a61cb1a7944689734ee4c5f467858 [ 744.355973] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg 0e411b73136543b29f90c1176f01c972 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 744.388372] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e411b73136543b29f90c1176f01c972 [ 744.403895] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.666998] env[64020]: DEBUG nova.compute.utils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 744.667670] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg f5b55601fc3a49818d0f2a8f100248d6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 744.669635] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg 4acce53bab344ba3834c45229fcaa017 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 744.670738] env[64020]: DEBUG nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 744.670911] env[64020]: DEBUG nova.network.neutron [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 744.682798] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5b55601fc3a49818d0f2a8f100248d6 [ 744.683820] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4acce53bab344ba3834c45229fcaa017 [ 744.728098] env[64020]: DEBUG nova.policy [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fc309fb9d0604703a2c159d2f52affe3', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e9f7ec1e87ee49bc9c39245e2d99c402', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 744.880155] env[64020]: INFO nova.scheduler.client.report [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Deleted allocations for instance 727e7787-68b7-441c-a31d-6f0796c33be0 [ 744.888186] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Expecting reply to msg cd8f8227bbde45738e80a6f6434e1936 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 744.900248] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd8f8227bbde45738e80a6f6434e1936 [ 745.012697] env[64020]: DEBUG nova.network.neutron [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Successfully created port: 991f8f50-f08a-4658-a355-5e0a04dc614f {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 745.173910] env[64020]: DEBUG nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 745.175633] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 0b2504c6e5154571973a9545cb24e075 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 745.220785] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b2504c6e5154571973a9545cb24e075 [ 745.315751] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "dcb3a0b0-a558-45be-8208-038e465dc802" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 745.316065] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "dcb3a0b0-a558-45be-8208-038e465dc802" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 745.389985] env[64020]: DEBUG oslo_concurrency.lockutils [None req-efe4efa3-a90d-4f3a-85fa-2932ebe7cf48 tempest-AttachInterfacesV270Test-1714307567 tempest-AttachInterfacesV270Test-1714307567-project-member] Lock "727e7787-68b7-441c-a31d-6f0796c33be0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 120.727s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 745.390750] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 146bf355409f4687b23b010261a7d342 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 745.400598] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 146bf355409f4687b23b010261a7d342 [ 745.592485] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78afb905-669d-466f-a3a5-7df57a3b7bb1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.601079] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62b86f42-5382-4be8-a16b-5281eac58bd7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.632047] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-812eb1cc-26a0-4156-8972-60c6ff862b9b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.641311] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c75b85cf-8a98-490c-bcbf-0d3564cc13d1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.655132] env[64020]: DEBUG nova.compute.provider_tree [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 745.656022] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg a6c0ea7a2a154659aa3d37ae268cb47b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 745.663813] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6c0ea7a2a154659aa3d37ae268cb47b [ 745.684420] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg caddf74aff674fefa12496538c9b94ef in queue reply_57893177120949e6a93cb88e15cd42b4 [ 745.691278] env[64020]: DEBUG nova.compute.manager [req-98871e9a-955c-425c-bfeb-58f6d760afe2 req-d06e4b2f-0ba4-4bbc-b8ca-6ff17da1987e service nova] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Received event network-changed-991f8f50-f08a-4658-a355-5e0a04dc614f {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 745.691278] env[64020]: DEBUG nova.compute.manager [req-98871e9a-955c-425c-bfeb-58f6d760afe2 req-d06e4b2f-0ba4-4bbc-b8ca-6ff17da1987e service nova] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Refreshing instance network info cache due to event network-changed-991f8f50-f08a-4658-a355-5e0a04dc614f. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 745.691278] env[64020]: DEBUG oslo_concurrency.lockutils [req-98871e9a-955c-425c-bfeb-58f6d760afe2 req-d06e4b2f-0ba4-4bbc-b8ca-6ff17da1987e service nova] Acquiring lock "refresh_cache-866f8ba4-faad-4883-80ca-4c4dedb2b50c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 745.691278] env[64020]: DEBUG oslo_concurrency.lockutils [req-98871e9a-955c-425c-bfeb-58f6d760afe2 req-d06e4b2f-0ba4-4bbc-b8ca-6ff17da1987e service nova] Acquired lock "refresh_cache-866f8ba4-faad-4883-80ca-4c4dedb2b50c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 745.691278] env[64020]: DEBUG nova.network.neutron [req-98871e9a-955c-425c-bfeb-58f6d760afe2 req-d06e4b2f-0ba4-4bbc-b8ca-6ff17da1987e service nova] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Refreshing network info cache for port 991f8f50-f08a-4658-a355-5e0a04dc614f {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 745.691521] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-98871e9a-955c-425c-bfeb-58f6d760afe2 req-d06e4b2f-0ba4-4bbc-b8ca-6ff17da1987e service nova] Expecting reply to msg de5135f8f2854d17abeabf87e2d0f9ed in queue reply_57893177120949e6a93cb88e15cd42b4 [ 745.697634] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de5135f8f2854d17abeabf87e2d0f9ed [ 745.717093] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg caddf74aff674fefa12496538c9b94ef [ 745.875332] env[64020]: ERROR nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 991f8f50-f08a-4658-a355-5e0a04dc614f, please check neutron logs for more information. [ 745.875332] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 745.875332] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 745.875332] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 745.875332] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 745.875332] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 745.875332] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 745.875332] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 745.875332] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 745.875332] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 745.875332] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 745.875332] env[64020]: ERROR nova.compute.manager raise self.value [ 745.875332] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 745.875332] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 745.875332] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 745.875332] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 745.876439] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 745.876439] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 745.876439] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 991f8f50-f08a-4658-a355-5e0a04dc614f, please check neutron logs for more information. [ 745.876439] env[64020]: ERROR nova.compute.manager [ 745.876439] env[64020]: Traceback (most recent call last): [ 745.876439] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 745.876439] env[64020]: listener.cb(fileno) [ 745.876439] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 745.876439] env[64020]: result = function(*args, **kwargs) [ 745.876439] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 745.876439] env[64020]: return func(*args, **kwargs) [ 745.876439] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 745.876439] env[64020]: raise e [ 745.876439] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 745.876439] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 745.876439] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 745.876439] env[64020]: created_port_ids = self._update_ports_for_instance( [ 745.876439] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 745.876439] env[64020]: with excutils.save_and_reraise_exception(): [ 745.876439] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 745.876439] env[64020]: self.force_reraise() [ 745.876439] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 745.876439] env[64020]: raise self.value [ 745.876439] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 745.876439] env[64020]: updated_port = self._update_port( [ 745.876439] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 745.876439] env[64020]: _ensure_no_port_binding_failure(port) [ 745.876439] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 745.876439] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 745.878117] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 991f8f50-f08a-4658-a355-5e0a04dc614f, please check neutron logs for more information. [ 745.878117] env[64020]: Removing descriptor: 18 [ 745.893406] env[64020]: DEBUG nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 745.895558] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 308d706c57304802ae04152a91365e7b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 745.945676] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 308d706c57304802ae04152a91365e7b [ 746.159447] env[64020]: DEBUG nova.scheduler.client.report [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 746.161989] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg 6675db8310204ee286839b6d887c85a9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 746.173277] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6675db8310204ee286839b6d887c85a9 [ 746.189990] env[64020]: DEBUG nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 746.210545] env[64020]: DEBUG nova.network.neutron [req-98871e9a-955c-425c-bfeb-58f6d760afe2 req-d06e4b2f-0ba4-4bbc-b8ca-6ff17da1987e service nova] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 746.218373] env[64020]: DEBUG nova.virt.hardware [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:26:38Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='260c3f58-30d6-43dc-8402-0fd60d28b999',id=36,is_public=True,memory_mb=192,name='tempest-test_resize_flavor_-807508408',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 746.218602] env[64020]: DEBUG nova.virt.hardware [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 746.218755] env[64020]: DEBUG nova.virt.hardware [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 746.218936] env[64020]: DEBUG nova.virt.hardware [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 746.219080] env[64020]: DEBUG nova.virt.hardware [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 746.219467] env[64020]: DEBUG nova.virt.hardware [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 746.219720] env[64020]: DEBUG nova.virt.hardware [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 746.220481] env[64020]: DEBUG nova.virt.hardware [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 746.220481] env[64020]: DEBUG nova.virt.hardware [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 746.220481] env[64020]: DEBUG nova.virt.hardware [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 746.220481] env[64020]: DEBUG nova.virt.hardware [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 746.221500] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e0e27dd-52bf-45be-876b-4ef926e04385 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.244028] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e527171-0d34-42cb-af2b-9c062ee2c350 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.258233] env[64020]: ERROR nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 991f8f50-f08a-4658-a355-5e0a04dc614f, please check neutron logs for more information. [ 746.258233] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Traceback (most recent call last): [ 746.258233] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 746.258233] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] yield resources [ 746.258233] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 746.258233] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] self.driver.spawn(context, instance, image_meta, [ 746.258233] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 746.258233] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 746.258233] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 746.258233] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] vm_ref = self.build_virtual_machine(instance, [ 746.258233] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] vif_infos = vmwarevif.get_vif_info(self._session, [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] for vif in network_info: [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] return self._sync_wrapper(fn, *args, **kwargs) [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] self.wait() [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] self[:] = self._gt.wait() [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] return self._exit_event.wait() [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 746.258683] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] current.throw(*self._exc) [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] result = function(*args, **kwargs) [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] return func(*args, **kwargs) [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] raise e [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] nwinfo = self.network_api.allocate_for_instance( [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] created_port_ids = self._update_ports_for_instance( [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] with excutils.save_and_reraise_exception(): [ 746.259046] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 746.259407] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] self.force_reraise() [ 746.259407] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 746.259407] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] raise self.value [ 746.259407] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 746.259407] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] updated_port = self._update_port( [ 746.259407] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 746.259407] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] _ensure_no_port_binding_failure(port) [ 746.259407] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 746.259407] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] raise exception.PortBindingFailed(port_id=port['id']) [ 746.259407] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] nova.exception.PortBindingFailed: Binding failed for port 991f8f50-f08a-4658-a355-5e0a04dc614f, please check neutron logs for more information. [ 746.259407] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] [ 746.259407] env[64020]: INFO nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Terminating instance [ 746.260964] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquiring lock "refresh_cache-866f8ba4-faad-4883-80ca-4c4dedb2b50c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 746.288456] env[64020]: DEBUG nova.network.neutron [req-98871e9a-955c-425c-bfeb-58f6d760afe2 req-d06e4b2f-0ba4-4bbc-b8ca-6ff17da1987e service nova] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 746.289116] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-98871e9a-955c-425c-bfeb-58f6d760afe2 req-d06e4b2f-0ba4-4bbc-b8ca-6ff17da1987e service nova] Expecting reply to msg 3b12b69d5ab946f683985fcd2969df57 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 746.297509] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b12b69d5ab946f683985fcd2969df57 [ 746.417490] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.664653] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.503s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 746.665160] env[64020]: DEBUG nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 746.667295] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg 7d4d8f5261ce4ea4925172d4a68fd386 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 746.668331] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 30.276s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.677602] env[64020]: INFO nova.compute.claims [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 746.677602] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 21d8ff2cccfa43f28a79ea4c97690408 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 746.707395] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d4d8f5261ce4ea4925172d4a68fd386 [ 746.713778] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21d8ff2cccfa43f28a79ea4c97690408 [ 746.791411] env[64020]: DEBUG oslo_concurrency.lockutils [req-98871e9a-955c-425c-bfeb-58f6d760afe2 req-d06e4b2f-0ba4-4bbc-b8ca-6ff17da1987e service nova] Releasing lock "refresh_cache-866f8ba4-faad-4883-80ca-4c4dedb2b50c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 746.791835] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquired lock "refresh_cache-866f8ba4-faad-4883-80ca-4c4dedb2b50c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 746.792118] env[64020]: DEBUG nova.network.neutron [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 746.792588] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg a05557d07c60499f814ffedbc0362a60 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 746.800517] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a05557d07c60499f814ffedbc0362a60 [ 747.178089] env[64020]: DEBUG nova.compute.utils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 747.178089] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg f5f4b9e8c59342229852385e912c3600 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 747.180588] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg fb181d55d89e4f44a89a0785ef107dbd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 747.180588] env[64020]: DEBUG nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 747.180588] env[64020]: DEBUG nova.network.neutron [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 747.196281] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb181d55d89e4f44a89a0785ef107dbd [ 747.196774] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5f4b9e8c59342229852385e912c3600 [ 747.272829] env[64020]: DEBUG nova.policy [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '578c04e7e9654a0393bb24bff5e0b7bc', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '67af57527c3741acb9ec59bc254ee045', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 747.327625] env[64020]: DEBUG nova.network.neutron [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 747.383519] env[64020]: DEBUG nova.network.neutron [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 747.384086] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg e22638a01e264d36a80322438b60b15b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 747.394437] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e22638a01e264d36a80322438b60b15b [ 747.564987] env[64020]: DEBUG nova.network.neutron [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Successfully created port: 1f4c9a60-7cd0-4a45-be03-9f0bb35b2353 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 747.681346] env[64020]: DEBUG nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 747.683100] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg cbac65b15ade444382d12624b1dedb32 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 747.721374] env[64020]: DEBUG nova.compute.manager [req-bcfe6460-6c9c-45da-a393-cec4d5afc186 req-5aaae4cd-9beb-44cd-a6ce-9e6b091c8a82 service nova] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Received event network-vif-deleted-991f8f50-f08a-4658-a355-5e0a04dc614f {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 747.727948] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cbac65b15ade444382d12624b1dedb32 [ 747.886256] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Releasing lock "refresh_cache-866f8ba4-faad-4883-80ca-4c4dedb2b50c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 747.886932] env[64020]: DEBUG nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 747.887136] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 747.887647] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-26823078-a920-4738-936e-111edb9feba3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 747.898751] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37173288-8e48-4657-8d7b-aec4e41ad944 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 747.923541] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 866f8ba4-faad-4883-80ca-4c4dedb2b50c could not be found. [ 747.923848] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 747.924068] env[64020]: INFO nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Took 0.04 seconds to destroy the instance on the hypervisor. [ 747.924324] env[64020]: DEBUG oslo.service.loopingcall [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 747.926732] env[64020]: DEBUG nova.compute.manager [-] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 747.926780] env[64020]: DEBUG nova.network.neutron [-] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 747.950061] env[64020]: DEBUG nova.network.neutron [-] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 747.950061] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 24d9ba92cdea47a4bc36d1e9a69e9905 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 747.960100] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 24d9ba92cdea47a4bc36d1e9a69e9905 [ 748.133981] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37a4378e-cc5d-4155-ad89-8f239b047f22 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.141711] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fbe6715e-d36c-4f8f-b4b3-a0d2df198067 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.182772] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39aaf9f8-404d-4287-9484-a046aa2eb855 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.189126] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg be1f435fb17b4574a1fffb55241cf338 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 748.197439] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d76a7d54-1174-46fa-b310-1fd08f462a17 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.214364] env[64020]: DEBUG nova.compute.provider_tree [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 748.215155] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 739b34d92bbf40d5a83d2845e6ae9d39 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 748.225185] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 739b34d92bbf40d5a83d2845e6ae9d39 [ 748.231728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be1f435fb17b4574a1fffb55241cf338 [ 748.451991] env[64020]: DEBUG nova.network.neutron [-] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 748.452625] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 30e371fe9baf44009659d080d4c427b7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 748.461710] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30e371fe9baf44009659d080d4c427b7 [ 748.593460] env[64020]: ERROR nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1f4c9a60-7cd0-4a45-be03-9f0bb35b2353, please check neutron logs for more information. [ 748.593460] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 748.593460] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.593460] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 748.593460] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 748.593460] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 748.593460] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 748.593460] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 748.593460] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.593460] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 748.593460] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.593460] env[64020]: ERROR nova.compute.manager raise self.value [ 748.593460] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 748.593460] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 748.593460] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.593460] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 748.593922] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.593922] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 748.593922] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1f4c9a60-7cd0-4a45-be03-9f0bb35b2353, please check neutron logs for more information. [ 748.593922] env[64020]: ERROR nova.compute.manager [ 748.593922] env[64020]: Traceback (most recent call last): [ 748.593922] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 748.593922] env[64020]: listener.cb(fileno) [ 748.593922] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 748.593922] env[64020]: result = function(*args, **kwargs) [ 748.593922] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 748.593922] env[64020]: return func(*args, **kwargs) [ 748.593922] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 748.593922] env[64020]: raise e [ 748.593922] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.593922] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 748.593922] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 748.593922] env[64020]: created_port_ids = self._update_ports_for_instance( [ 748.593922] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 748.593922] env[64020]: with excutils.save_and_reraise_exception(): [ 748.593922] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.593922] env[64020]: self.force_reraise() [ 748.593922] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.593922] env[64020]: raise self.value [ 748.593922] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 748.593922] env[64020]: updated_port = self._update_port( [ 748.593922] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.593922] env[64020]: _ensure_no_port_binding_failure(port) [ 748.593922] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.593922] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 748.594611] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 1f4c9a60-7cd0-4a45-be03-9f0bb35b2353, please check neutron logs for more information. [ 748.594611] env[64020]: Removing descriptor: 18 [ 748.692293] env[64020]: DEBUG nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 748.717949] env[64020]: DEBUG nova.virt.hardware [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 748.718239] env[64020]: DEBUG nova.virt.hardware [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 748.718526] env[64020]: DEBUG nova.virt.hardware [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 748.718681] env[64020]: DEBUG nova.virt.hardware [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 748.718858] env[64020]: DEBUG nova.virt.hardware [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 748.719049] env[64020]: DEBUG nova.virt.hardware [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 748.719326] env[64020]: DEBUG nova.virt.hardware [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 748.719524] env[64020]: DEBUG nova.virt.hardware [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 748.719727] env[64020]: DEBUG nova.virt.hardware [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 748.719916] env[64020]: DEBUG nova.virt.hardware [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 748.720132] env[64020]: DEBUG nova.virt.hardware [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 748.721324] env[64020]: DEBUG nova.scheduler.client.report [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 748.723697] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg b8a59862d8b849c790249f9991cf3ad5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 748.725332] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28fe99aa-4dd1-4c74-984c-b95b4c351da3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.733300] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-783deb5a-9e51-4747-bbf7-55e79d3fbaca {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.737415] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8a59862d8b849c790249f9991cf3ad5 [ 748.748014] env[64020]: ERROR nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1f4c9a60-7cd0-4a45-be03-9f0bb35b2353, please check neutron logs for more information. [ 748.748014] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Traceback (most recent call last): [ 748.748014] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 748.748014] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] yield resources [ 748.748014] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 748.748014] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] self.driver.spawn(context, instance, image_meta, [ 748.748014] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 748.748014] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 748.748014] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 748.748014] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] vm_ref = self.build_virtual_machine(instance, [ 748.748014] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] vif_infos = vmwarevif.get_vif_info(self._session, [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] for vif in network_info: [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] return self._sync_wrapper(fn, *args, **kwargs) [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] self.wait() [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] self[:] = self._gt.wait() [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] return self._exit_event.wait() [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 748.748589] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] current.throw(*self._exc) [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] result = function(*args, **kwargs) [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] return func(*args, **kwargs) [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] raise e [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] nwinfo = self.network_api.allocate_for_instance( [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] created_port_ids = self._update_ports_for_instance( [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] with excutils.save_and_reraise_exception(): [ 748.749269] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.749744] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] self.force_reraise() [ 748.749744] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.749744] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] raise self.value [ 748.749744] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 748.749744] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] updated_port = self._update_port( [ 748.749744] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.749744] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] _ensure_no_port_binding_failure(port) [ 748.749744] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.749744] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] raise exception.PortBindingFailed(port_id=port['id']) [ 748.749744] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] nova.exception.PortBindingFailed: Binding failed for port 1f4c9a60-7cd0-4a45-be03-9f0bb35b2353, please check neutron logs for more information. [ 748.749744] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] [ 748.749744] env[64020]: INFO nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Terminating instance [ 748.758902] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Acquiring lock "refresh_cache-371d5fb1-f1b6-41de-a4e6-88e8137f4be9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 748.758902] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Acquired lock "refresh_cache-371d5fb1-f1b6-41de-a4e6-88e8137f4be9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 748.758902] env[64020]: DEBUG nova.network.neutron [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 748.758902] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg 30bfd3f2bcec4f12a9a844a0b458b286 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 748.758902] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30bfd3f2bcec4f12a9a844a0b458b286 [ 748.954570] env[64020]: INFO nova.compute.manager [-] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Took 1.03 seconds to deallocate network for instance. [ 748.957151] env[64020]: DEBUG nova.compute.claims [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 748.957363] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 749.230050] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.561s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 749.230050] env[64020]: DEBUG nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 749.231845] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 222ed0e3ab4e47368d69846ec19fc4f4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 749.232910] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 30.801s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 749.234249] env[64020]: INFO nova.compute.claims [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 749.235722] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 4fd35983ee76443dbe152feb5f0cea80 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 749.269054] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 222ed0e3ab4e47368d69846ec19fc4f4 [ 749.273562] env[64020]: DEBUG nova.network.neutron [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 749.291220] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4fd35983ee76443dbe152feb5f0cea80 [ 749.522338] env[64020]: DEBUG nova.network.neutron [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 749.522868] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg c48abba9c76247adab7494de5c9fe1f8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 749.531865] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c48abba9c76247adab7494de5c9fe1f8 [ 749.738568] env[64020]: DEBUG nova.compute.utils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 749.739240] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg e657c5a4c70c417caa44420387323ab5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 749.741287] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 86635ba2f39d48c6a806e3528aa813cc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 749.742081] env[64020]: DEBUG nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 749.742275] env[64020]: DEBUG nova.network.neutron [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 749.749630] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e657c5a4c70c417caa44420387323ab5 [ 749.752238] env[64020]: DEBUG nova.compute.manager [req-0fac5efe-8e6e-466d-8788-ef15ba9252a4 req-d816ae01-5309-4017-b0f1-be8125b68421 service nova] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Received event network-changed-1f4c9a60-7cd0-4a45-be03-9f0bb35b2353 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 749.752413] env[64020]: DEBUG nova.compute.manager [req-0fac5efe-8e6e-466d-8788-ef15ba9252a4 req-d816ae01-5309-4017-b0f1-be8125b68421 service nova] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Refreshing instance network info cache due to event network-changed-1f4c9a60-7cd0-4a45-be03-9f0bb35b2353. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 749.752595] env[64020]: DEBUG oslo_concurrency.lockutils [req-0fac5efe-8e6e-466d-8788-ef15ba9252a4 req-d816ae01-5309-4017-b0f1-be8125b68421 service nova] Acquiring lock "refresh_cache-371d5fb1-f1b6-41de-a4e6-88e8137f4be9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 749.755001] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86635ba2f39d48c6a806e3528aa813cc [ 749.818163] env[64020]: DEBUG nova.policy [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2e612db51c8a49b3b942ac70e24025f6', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd139e605a6f54506ab7b94cc56578734', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 750.025134] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Releasing lock "refresh_cache-371d5fb1-f1b6-41de-a4e6-88e8137f4be9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 750.025568] env[64020]: DEBUG nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 750.025777] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 750.026099] env[64020]: DEBUG oslo_concurrency.lockutils [req-0fac5efe-8e6e-466d-8788-ef15ba9252a4 req-d816ae01-5309-4017-b0f1-be8125b68421 service nova] Acquired lock "refresh_cache-371d5fb1-f1b6-41de-a4e6-88e8137f4be9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 750.026268] env[64020]: DEBUG nova.network.neutron [req-0fac5efe-8e6e-466d-8788-ef15ba9252a4 req-d816ae01-5309-4017-b0f1-be8125b68421 service nova] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Refreshing network info cache for port 1f4c9a60-7cd0-4a45-be03-9f0bb35b2353 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 750.026771] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-0fac5efe-8e6e-466d-8788-ef15ba9252a4 req-d816ae01-5309-4017-b0f1-be8125b68421 service nova] Expecting reply to msg 0e6393a0d6a741fd8e55365f915a5e16 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 750.027552] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-dfdc3c0f-c299-4f0c-8f59-7dda23404a63 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.036828] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e54cc64-57d0-4895-9881-60958142d732 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.048578] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e6393a0d6a741fd8e55365f915a5e16 [ 750.061055] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 371d5fb1-f1b6-41de-a4e6-88e8137f4be9 could not be found. [ 750.061319] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 750.061494] env[64020]: INFO nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Took 0.04 seconds to destroy the instance on the hypervisor. [ 750.061727] env[64020]: DEBUG oslo.service.loopingcall [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 750.061933] env[64020]: DEBUG nova.compute.manager [-] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 750.062020] env[64020]: DEBUG nova.network.neutron [-] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 750.078907] env[64020]: DEBUG nova.network.neutron [-] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 750.079531] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 854dba59ec8d484e9c905ed8407b08c7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 750.087728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 854dba59ec8d484e9c905ed8407b08c7 [ 750.174514] env[64020]: DEBUG nova.network.neutron [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Successfully created port: f27b0ef4-53c6-4a55-be67-1ced3268cfbc {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 750.248335] env[64020]: DEBUG nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 750.250156] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 09d4765a6e144981bb6ca17654f3316f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 750.295481] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 09d4765a6e144981bb6ca17654f3316f [ 750.549541] env[64020]: DEBUG nova.network.neutron [req-0fac5efe-8e6e-466d-8788-ef15ba9252a4 req-d816ae01-5309-4017-b0f1-be8125b68421 service nova] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 750.581952] env[64020]: DEBUG nova.network.neutron [-] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 750.582388] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c547c040cf384646b6b38aced05ca633 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 750.599886] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c547c040cf384646b6b38aced05ca633 [ 750.647744] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55151ec9-7e19-4891-aa5d-f745895b56d8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.665103] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38701f02-5159-44bc-a4dc-29d5cbf3e276 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.696693] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e150ff5-2a7b-40a1-a783-bf22600d2496 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.707607] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff73e761-88fe-4a84-a153-f3bea41b4e45 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.731792] env[64020]: DEBUG nova.compute.provider_tree [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 750.732322] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 1a764e1dd4c345e690225be77e4cb1c7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 750.734001] env[64020]: DEBUG nova.network.neutron [req-0fac5efe-8e6e-466d-8788-ef15ba9252a4 req-d816ae01-5309-4017-b0f1-be8125b68421 service nova] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 750.734450] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-0fac5efe-8e6e-466d-8788-ef15ba9252a4 req-d816ae01-5309-4017-b0f1-be8125b68421 service nova] Expecting reply to msg e6d748f4fa4f43248dd001c1e2225aba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 750.745696] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a764e1dd4c345e690225be77e4cb1c7 [ 750.746201] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6d748f4fa4f43248dd001c1e2225aba [ 750.754725] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 5ebf09fb0c96443fa86f4fb7dc995e48 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 750.784689] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ebf09fb0c96443fa86f4fb7dc995e48 [ 751.088865] env[64020]: INFO nova.compute.manager [-] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Took 1.03 seconds to deallocate network for instance. [ 751.091331] env[64020]: DEBUG nova.compute.claims [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 751.091506] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 751.205676] env[64020]: ERROR nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f27b0ef4-53c6-4a55-be67-1ced3268cfbc, please check neutron logs for more information. [ 751.205676] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 751.205676] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 751.205676] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 751.205676] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 751.205676] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 751.205676] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 751.205676] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 751.205676] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 751.205676] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 751.205676] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 751.205676] env[64020]: ERROR nova.compute.manager raise self.value [ 751.205676] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 751.205676] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 751.205676] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 751.205676] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 751.206275] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 751.206275] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 751.206275] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f27b0ef4-53c6-4a55-be67-1ced3268cfbc, please check neutron logs for more information. [ 751.206275] env[64020]: ERROR nova.compute.manager [ 751.206275] env[64020]: Traceback (most recent call last): [ 751.206275] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 751.206275] env[64020]: listener.cb(fileno) [ 751.206275] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 751.206275] env[64020]: result = function(*args, **kwargs) [ 751.206275] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 751.206275] env[64020]: return func(*args, **kwargs) [ 751.206275] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 751.206275] env[64020]: raise e [ 751.206275] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 751.206275] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 751.206275] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 751.206275] env[64020]: created_port_ids = self._update_ports_for_instance( [ 751.206275] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 751.206275] env[64020]: with excutils.save_and_reraise_exception(): [ 751.206275] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 751.206275] env[64020]: self.force_reraise() [ 751.206275] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 751.206275] env[64020]: raise self.value [ 751.206275] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 751.206275] env[64020]: updated_port = self._update_port( [ 751.206275] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 751.206275] env[64020]: _ensure_no_port_binding_failure(port) [ 751.206275] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 751.206275] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 751.207199] env[64020]: nova.exception.PortBindingFailed: Binding failed for port f27b0ef4-53c6-4a55-be67-1ced3268cfbc, please check neutron logs for more information. [ 751.207199] env[64020]: Removing descriptor: 18 [ 751.218781] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Acquiring lock "c9a9eccd-4730-4815-8270-8272ac8bdec6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 751.219010] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Lock "c9a9eccd-4730-4815-8270-8272ac8bdec6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 751.238955] env[64020]: DEBUG nova.scheduler.client.report [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 751.238955] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 8a51a9a9e8684e048ffc3596b8b4d91e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 751.245289] env[64020]: DEBUG oslo_concurrency.lockutils [req-0fac5efe-8e6e-466d-8788-ef15ba9252a4 req-d816ae01-5309-4017-b0f1-be8125b68421 service nova] Releasing lock "refresh_cache-371d5fb1-f1b6-41de-a4e6-88e8137f4be9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 751.245518] env[64020]: DEBUG nova.compute.manager [req-0fac5efe-8e6e-466d-8788-ef15ba9252a4 req-d816ae01-5309-4017-b0f1-be8125b68421 service nova] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Received event network-vif-deleted-1f4c9a60-7cd0-4a45-be03-9f0bb35b2353 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 751.249029] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a51a9a9e8684e048ffc3596b8b4d91e [ 751.258145] env[64020]: DEBUG nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 751.281970] env[64020]: DEBUG nova.virt.hardware [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 751.282276] env[64020]: DEBUG nova.virt.hardware [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 751.282478] env[64020]: DEBUG nova.virt.hardware [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 751.282699] env[64020]: DEBUG nova.virt.hardware [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 751.282879] env[64020]: DEBUG nova.virt.hardware [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 751.283056] env[64020]: DEBUG nova.virt.hardware [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 751.283298] env[64020]: DEBUG nova.virt.hardware [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 751.283491] env[64020]: DEBUG nova.virt.hardware [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 751.283690] env[64020]: DEBUG nova.virt.hardware [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 751.283884] env[64020]: DEBUG nova.virt.hardware [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 751.284107] env[64020]: DEBUG nova.virt.hardware [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 751.285257] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94dcb79b-202d-4608-ad5f-f841dbc9a0f6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.294227] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc754805-1d35-4e5f-9ced-9547a27a866e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.308574] env[64020]: ERROR nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f27b0ef4-53c6-4a55-be67-1ced3268cfbc, please check neutron logs for more information. [ 751.308574] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Traceback (most recent call last): [ 751.308574] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 751.308574] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] yield resources [ 751.308574] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 751.308574] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] self.driver.spawn(context, instance, image_meta, [ 751.308574] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 751.308574] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 751.308574] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 751.308574] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] vm_ref = self.build_virtual_machine(instance, [ 751.308574] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] vif_infos = vmwarevif.get_vif_info(self._session, [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] for vif in network_info: [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] return self._sync_wrapper(fn, *args, **kwargs) [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] self.wait() [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] self[:] = self._gt.wait() [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] return self._exit_event.wait() [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 751.308996] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] current.throw(*self._exc) [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] result = function(*args, **kwargs) [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] return func(*args, **kwargs) [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] raise e [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] nwinfo = self.network_api.allocate_for_instance( [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] created_port_ids = self._update_ports_for_instance( [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] with excutils.save_and_reraise_exception(): [ 751.309354] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 751.309713] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] self.force_reraise() [ 751.309713] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 751.309713] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] raise self.value [ 751.309713] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 751.309713] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] updated_port = self._update_port( [ 751.309713] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 751.309713] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] _ensure_no_port_binding_failure(port) [ 751.309713] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 751.309713] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] raise exception.PortBindingFailed(port_id=port['id']) [ 751.309713] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] nova.exception.PortBindingFailed: Binding failed for port f27b0ef4-53c6-4a55-be67-1ced3268cfbc, please check neutron logs for more information. [ 751.309713] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] [ 751.309713] env[64020]: INFO nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Terminating instance [ 751.311684] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "refresh_cache-a26baa75-d36d-4c05-bde8-d39fe814191d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 751.311996] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquired lock "refresh_cache-a26baa75-d36d-4c05-bde8-d39fe814191d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 751.312232] env[64020]: DEBUG nova.network.neutron [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 751.312691] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 2bced85fc2e7451f9abf5fc15ba2d5a5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 751.318816] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2bced85fc2e7451f9abf5fc15ba2d5a5 [ 751.741063] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.508s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 751.741670] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 751.743402] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg f6755203b187413182d28334b6daa5b5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 751.744407] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 31.277s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 751.745779] env[64020]: INFO nova.compute.claims [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 751.747596] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg fe9373308a9b49748ab35b2beb1579d1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 751.792118] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f6755203b187413182d28334b6daa5b5 [ 751.802760] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe9373308a9b49748ab35b2beb1579d1 [ 751.806311] env[64020]: DEBUG nova.compute.manager [req-dfddd40d-c46a-4129-b4c8-657903b31c1c req-99d2a742-c148-40f4-86c5-f3f4e90f1df5 service nova] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Received event network-changed-f27b0ef4-53c6-4a55-be67-1ced3268cfbc {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 751.806482] env[64020]: DEBUG nova.compute.manager [req-dfddd40d-c46a-4129-b4c8-657903b31c1c req-99d2a742-c148-40f4-86c5-f3f4e90f1df5 service nova] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Refreshing instance network info cache due to event network-changed-f27b0ef4-53c6-4a55-be67-1ced3268cfbc. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 751.806748] env[64020]: DEBUG oslo_concurrency.lockutils [req-dfddd40d-c46a-4129-b4c8-657903b31c1c req-99d2a742-c148-40f4-86c5-f3f4e90f1df5 service nova] Acquiring lock "refresh_cache-a26baa75-d36d-4c05-bde8-d39fe814191d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 751.828416] env[64020]: DEBUG nova.network.neutron [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 751.921894] env[64020]: DEBUG nova.network.neutron [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 751.922594] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg feea2a31968e4479854f198349f9e1e0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 751.931113] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg feea2a31968e4479854f198349f9e1e0 [ 752.250138] env[64020]: DEBUG nova.compute.utils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 752.250774] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg a9e3199ba9cf4b549b0445993e8d875b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 752.252823] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg f8fa156bc8624846b6d2efbcbdbe66a0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 752.259465] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 752.259465] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 752.260168] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f8fa156bc8624846b6d2efbcbdbe66a0 [ 752.260638] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a9e3199ba9cf4b549b0445993e8d875b [ 752.302477] env[64020]: DEBUG nova.policy [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '478fd14e8faf4a9e844bf53e9be91892', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5571feef12fc4ec6a09b15b69abeb4e5', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 752.425717] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Releasing lock "refresh_cache-a26baa75-d36d-4c05-bde8-d39fe814191d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 752.425717] env[64020]: DEBUG nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 752.425824] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 752.426122] env[64020]: DEBUG oslo_concurrency.lockutils [req-dfddd40d-c46a-4129-b4c8-657903b31c1c req-99d2a742-c148-40f4-86c5-f3f4e90f1df5 service nova] Acquired lock "refresh_cache-a26baa75-d36d-4c05-bde8-d39fe814191d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 752.426286] env[64020]: DEBUG nova.network.neutron [req-dfddd40d-c46a-4129-b4c8-657903b31c1c req-99d2a742-c148-40f4-86c5-f3f4e90f1df5 service nova] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Refreshing network info cache for port f27b0ef4-53c6-4a55-be67-1ced3268cfbc {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 752.426797] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-dfddd40d-c46a-4129-b4c8-657903b31c1c req-99d2a742-c148-40f4-86c5-f3f4e90f1df5 service nova] Expecting reply to msg 5014ee7853b6471b81c2d8821bfdab16 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 752.427562] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f40d7a51-3922-4f95-8eea-c1a4d95b19eb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 752.434329] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5014ee7853b6471b81c2d8821bfdab16 [ 752.437331] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8735b6b4-2813-42d6-a583-0499afe44bb8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 752.458443] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a26baa75-d36d-4c05-bde8-d39fe814191d could not be found. [ 752.458650] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 752.458826] env[64020]: INFO nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Took 0.03 seconds to destroy the instance on the hypervisor. [ 752.459074] env[64020]: DEBUG oslo.service.loopingcall [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 752.459280] env[64020]: DEBUG nova.compute.manager [-] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 752.459373] env[64020]: DEBUG nova.network.neutron [-] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 752.473505] env[64020]: DEBUG nova.network.neutron [-] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 752.473947] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7a530c1e786b4694bf7c072f5849196f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 752.480575] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a530c1e786b4694bf7c072f5849196f [ 752.571374] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Successfully created port: 829381ab-a31a-4ad8-a28d-ace00c45d6c8 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 752.755531] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 752.756763] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg a46347adf2084d90bf5e0499cfa1a109 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 752.811529] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a46347adf2084d90bf5e0499cfa1a109 [ 752.956868] env[64020]: DEBUG nova.network.neutron [req-dfddd40d-c46a-4129-b4c8-657903b31c1c req-99d2a742-c148-40f4-86c5-f3f4e90f1df5 service nova] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 752.975965] env[64020]: DEBUG nova.network.neutron [-] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 752.976444] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 647d7c245ae8488cbe4408bb6b7e8c30 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 752.985864] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 647d7c245ae8488cbe4408bb6b7e8c30 [ 753.095902] env[64020]: DEBUG nova.network.neutron [req-dfddd40d-c46a-4129-b4c8-657903b31c1c req-99d2a742-c148-40f4-86c5-f3f4e90f1df5 service nova] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 753.096447] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-dfddd40d-c46a-4129-b4c8-657903b31c1c req-99d2a742-c148-40f4-86c5-f3f4e90f1df5 service nova] Expecting reply to msg 98378a834d6c4c7fa6a8cde5b64dfd31 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 753.105416] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98378a834d6c4c7fa6a8cde5b64dfd31 [ 753.181088] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf70e1bf-22f2-49a8-98ef-fa677f1f1a63 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.189197] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb3bb1ab-9406-451e-9ade-6c1ae7f414a6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.222284] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a71c1d7-833f-4989-be01-cf1de6328df9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.230675] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5360d144-2c22-4138-a183-dac704487560 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.246168] env[64020]: DEBUG nova.compute.provider_tree [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 753.246700] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 92b2f57a88db4dd9a40d4d4557bca53e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 753.254097] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92b2f57a88db4dd9a40d4d4557bca53e [ 753.267324] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 8804102499d242b6a77df144f4097c97 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 753.308586] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8804102499d242b6a77df144f4097c97 [ 753.481417] env[64020]: INFO nova.compute.manager [-] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Took 1.02 seconds to deallocate network for instance. [ 753.484203] env[64020]: DEBUG nova.compute.claims [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 753.484471] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 753.538374] env[64020]: ERROR nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 829381ab-a31a-4ad8-a28d-ace00c45d6c8, please check neutron logs for more information. [ 753.538374] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 753.538374] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 753.538374] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 753.538374] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 753.538374] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 753.538374] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 753.538374] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 753.538374] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 753.538374] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 753.538374] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 753.538374] env[64020]: ERROR nova.compute.manager raise self.value [ 753.538374] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 753.538374] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 753.538374] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 753.538374] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 753.538840] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 753.538840] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 753.538840] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 829381ab-a31a-4ad8-a28d-ace00c45d6c8, please check neutron logs for more information. [ 753.538840] env[64020]: ERROR nova.compute.manager [ 753.538840] env[64020]: Traceback (most recent call last): [ 753.538840] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 753.538840] env[64020]: listener.cb(fileno) [ 753.538840] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 753.538840] env[64020]: result = function(*args, **kwargs) [ 753.538840] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 753.538840] env[64020]: return func(*args, **kwargs) [ 753.538840] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 753.538840] env[64020]: raise e [ 753.538840] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 753.538840] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 753.538840] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 753.538840] env[64020]: created_port_ids = self._update_ports_for_instance( [ 753.538840] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 753.538840] env[64020]: with excutils.save_and_reraise_exception(): [ 753.538840] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 753.538840] env[64020]: self.force_reraise() [ 753.538840] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 753.538840] env[64020]: raise self.value [ 753.538840] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 753.538840] env[64020]: updated_port = self._update_port( [ 753.538840] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 753.538840] env[64020]: _ensure_no_port_binding_failure(port) [ 753.538840] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 753.538840] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 753.539681] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 829381ab-a31a-4ad8-a28d-ace00c45d6c8, please check neutron logs for more information. [ 753.539681] env[64020]: Removing descriptor: 18 [ 753.598901] env[64020]: DEBUG oslo_concurrency.lockutils [req-dfddd40d-c46a-4129-b4c8-657903b31c1c req-99d2a742-c148-40f4-86c5-f3f4e90f1df5 service nova] Releasing lock "refresh_cache-a26baa75-d36d-4c05-bde8-d39fe814191d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 753.599197] env[64020]: DEBUG nova.compute.manager [req-dfddd40d-c46a-4129-b4c8-657903b31c1c req-99d2a742-c148-40f4-86c5-f3f4e90f1df5 service nova] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Received event network-vif-deleted-f27b0ef4-53c6-4a55-be67-1ced3268cfbc {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 753.750230] env[64020]: DEBUG nova.scheduler.client.report [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 753.752723] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 128b9a0cbfee40e483e09971257a1f72 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 753.765294] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 128b9a0cbfee40e483e09971257a1f72 [ 753.773451] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 753.805172] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 753.805421] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 753.805571] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 753.805745] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 753.805883] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 753.806022] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 753.806215] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 753.806367] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 753.806548] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 753.806720] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 753.806886] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 753.807702] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f2ec5e5-8f57-489a-8c4e-ec7eff966239 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.816741] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4af87811-136a-448a-b864-aec51efbe7f6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.835757] env[64020]: ERROR nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 829381ab-a31a-4ad8-a28d-ace00c45d6c8, please check neutron logs for more information. [ 753.835757] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Traceback (most recent call last): [ 753.835757] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 753.835757] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] yield resources [ 753.835757] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 753.835757] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] self.driver.spawn(context, instance, image_meta, [ 753.835757] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 753.835757] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] self._vmops.spawn(context, instance, image_meta, injected_files, [ 753.835757] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 753.835757] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] vm_ref = self.build_virtual_machine(instance, [ 753.835757] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] vif_infos = vmwarevif.get_vif_info(self._session, [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] for vif in network_info: [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] return self._sync_wrapper(fn, *args, **kwargs) [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] self.wait() [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] self[:] = self._gt.wait() [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] return self._exit_event.wait() [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 753.836087] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] current.throw(*self._exc) [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] result = function(*args, **kwargs) [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] return func(*args, **kwargs) [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] raise e [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] nwinfo = self.network_api.allocate_for_instance( [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] created_port_ids = self._update_ports_for_instance( [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] with excutils.save_and_reraise_exception(): [ 753.836412] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 753.836751] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] self.force_reraise() [ 753.836751] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 753.836751] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] raise self.value [ 753.836751] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 753.836751] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] updated_port = self._update_port( [ 753.836751] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 753.836751] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] _ensure_no_port_binding_failure(port) [ 753.836751] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 753.836751] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] raise exception.PortBindingFailed(port_id=port['id']) [ 753.836751] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] nova.exception.PortBindingFailed: Binding failed for port 829381ab-a31a-4ad8-a28d-ace00c45d6c8, please check neutron logs for more information. [ 753.836751] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] [ 753.836751] env[64020]: INFO nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Terminating instance [ 753.839181] env[64020]: DEBUG nova.compute.manager [req-1a684856-fa79-47a4-9695-0eff8168130f req-39cd7594-fd6f-4f4e-9328-f4445afde8e3 service nova] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Received event network-changed-829381ab-a31a-4ad8-a28d-ace00c45d6c8 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 753.839364] env[64020]: DEBUG nova.compute.manager [req-1a684856-fa79-47a4-9695-0eff8168130f req-39cd7594-fd6f-4f4e-9328-f4445afde8e3 service nova] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Refreshing instance network info cache due to event network-changed-829381ab-a31a-4ad8-a28d-ace00c45d6c8. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 753.839563] env[64020]: DEBUG oslo_concurrency.lockutils [req-1a684856-fa79-47a4-9695-0eff8168130f req-39cd7594-fd6f-4f4e-9328-f4445afde8e3 service nova] Acquiring lock "refresh_cache-e01583c9-744c-47ea-b7cf-a46b046fc854" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 753.839698] env[64020]: DEBUG oslo_concurrency.lockutils [req-1a684856-fa79-47a4-9695-0eff8168130f req-39cd7594-fd6f-4f4e-9328-f4445afde8e3 service nova] Acquired lock "refresh_cache-e01583c9-744c-47ea-b7cf-a46b046fc854" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 753.839851] env[64020]: DEBUG nova.network.neutron [req-1a684856-fa79-47a4-9695-0eff8168130f req-39cd7594-fd6f-4f4e-9328-f4445afde8e3 service nova] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Refreshing network info cache for port 829381ab-a31a-4ad8-a28d-ace00c45d6c8 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 753.840255] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-1a684856-fa79-47a4-9695-0eff8168130f req-39cd7594-fd6f-4f4e-9328-f4445afde8e3 service nova] Expecting reply to msg a635b548d8194896b6fe9ce292e6b161 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 753.841098] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "refresh_cache-e01583c9-744c-47ea-b7cf-a46b046fc854" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 753.846769] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a635b548d8194896b6fe9ce292e6b161 [ 754.255129] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.511s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 754.255660] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 754.257371] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 100c09629b52484d85190a48fbe21fd0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 754.258679] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 30.480s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 754.260553] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 2b32b511ab3b46ca8a314efd89612194 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 754.297068] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b32b511ab3b46ca8a314efd89612194 [ 754.298065] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 100c09629b52484d85190a48fbe21fd0 [ 754.359652] env[64020]: DEBUG nova.network.neutron [req-1a684856-fa79-47a4-9695-0eff8168130f req-39cd7594-fd6f-4f4e-9328-f4445afde8e3 service nova] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 754.454512] env[64020]: DEBUG nova.network.neutron [req-1a684856-fa79-47a4-9695-0eff8168130f req-39cd7594-fd6f-4f4e-9328-f4445afde8e3 service nova] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 754.455037] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-1a684856-fa79-47a4-9695-0eff8168130f req-39cd7594-fd6f-4f4e-9328-f4445afde8e3 service nova] Expecting reply to msg ceac98dea24c49fd8987376fb10b40bf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 754.472962] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ceac98dea24c49fd8987376fb10b40bf [ 754.764433] env[64020]: DEBUG nova.compute.utils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 754.765048] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 82ba4f0b15a44175ab7c70799dce4237 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 754.769823] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 754.770022] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 754.775256] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82ba4f0b15a44175ab7c70799dce4237 [ 754.811008] env[64020]: DEBUG nova.policy [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '478fd14e8faf4a9e844bf53e9be91892', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5571feef12fc4ec6a09b15b69abeb4e5', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 754.957579] env[64020]: DEBUG oslo_concurrency.lockutils [req-1a684856-fa79-47a4-9695-0eff8168130f req-39cd7594-fd6f-4f4e-9328-f4445afde8e3 service nova] Releasing lock "refresh_cache-e01583c9-744c-47ea-b7cf-a46b046fc854" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 754.957826] env[64020]: DEBUG nova.compute.manager [req-1a684856-fa79-47a4-9695-0eff8168130f req-39cd7594-fd6f-4f4e-9328-f4445afde8e3 service nova] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Received event network-vif-deleted-829381ab-a31a-4ad8-a28d-ace00c45d6c8 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 754.958174] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquired lock "refresh_cache-e01583c9-744c-47ea-b7cf-a46b046fc854" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 754.958335] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 754.958769] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg ea77d6797a2443f9bb63060a46998297 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 754.966220] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ea77d6797a2443f9bb63060a46998297 [ 755.090092] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Successfully created port: c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 755.108371] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4fdae280-d272-456f-aa3e-5b1f022102b1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.117031] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-feb9952e-5e58-4f8b-9cf3-f43f5559ec0d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.148670] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-514340b5-5314-4b42-8ce2-f204d79e24de {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.156578] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2646ffe2-7c18-4972-a977-f41d1ac28b67 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.171578] env[64020]: DEBUG nova.compute.provider_tree [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 755.172108] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 20a3944af3324de4b5c8e4396a53623f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 755.179014] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20a3944af3324de4b5c8e4396a53623f [ 755.269405] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 755.271226] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 8b7d5ecc53124a68a1f76eef7ee1ec8a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 755.303475] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b7d5ecc53124a68a1f76eef7ee1ec8a [ 755.485103] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 755.581000] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 755.581666] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 07885c91b9d249409141d653f7e1cc95 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 755.589698] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 07885c91b9d249409141d653f7e1cc95 [ 755.675218] env[64020]: DEBUG nova.scheduler.client.report [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 755.677550] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 7a3956ee367747caa6f064011b4a8f9c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 755.691320] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a3956ee367747caa6f064011b4a8f9c [ 755.775984] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 2c05ef3f28754214aa8118df87b64852 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 755.805179] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c05ef3f28754214aa8118df87b64852 [ 755.861873] env[64020]: DEBUG nova.compute.manager [req-d561604b-04d8-4a5c-ad64-4f21fede526d req-4015d35e-59c9-4268-b841-f9126d31f380 service nova] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Received event network-changed-c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 755.861988] env[64020]: DEBUG nova.compute.manager [req-d561604b-04d8-4a5c-ad64-4f21fede526d req-4015d35e-59c9-4268-b841-f9126d31f380 service nova] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Refreshing instance network info cache due to event network-changed-c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 755.862435] env[64020]: DEBUG oslo_concurrency.lockutils [req-d561604b-04d8-4a5c-ad64-4f21fede526d req-4015d35e-59c9-4268-b841-f9126d31f380 service nova] Acquiring lock "refresh_cache-786a572c-adaa-4656-8b42-c510940dac59" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 755.862435] env[64020]: DEBUG oslo_concurrency.lockutils [req-d561604b-04d8-4a5c-ad64-4f21fede526d req-4015d35e-59c9-4268-b841-f9126d31f380 service nova] Acquired lock "refresh_cache-786a572c-adaa-4656-8b42-c510940dac59" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 755.862510] env[64020]: DEBUG nova.network.neutron [req-d561604b-04d8-4a5c-ad64-4f21fede526d req-4015d35e-59c9-4268-b841-f9126d31f380 service nova] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Refreshing network info cache for port c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 755.862893] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-d561604b-04d8-4a5c-ad64-4f21fede526d req-4015d35e-59c9-4268-b841-f9126d31f380 service nova] Expecting reply to msg 9c24d9292cce49e2be44bc5473d4b760 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 755.869713] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c24d9292cce49e2be44bc5473d4b760 [ 755.881700] env[64020]: ERROR nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8, please check neutron logs for more information. [ 755.881700] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 755.881700] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 755.881700] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 755.881700] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 755.881700] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 755.881700] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 755.881700] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 755.881700] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 755.881700] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 755.881700] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 755.881700] env[64020]: ERROR nova.compute.manager raise self.value [ 755.881700] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 755.881700] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 755.881700] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 755.881700] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 755.882223] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 755.882223] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 755.882223] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8, please check neutron logs for more information. [ 755.882223] env[64020]: ERROR nova.compute.manager [ 755.882223] env[64020]: Traceback (most recent call last): [ 755.882223] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 755.882223] env[64020]: listener.cb(fileno) [ 755.882223] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 755.882223] env[64020]: result = function(*args, **kwargs) [ 755.882223] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 755.882223] env[64020]: return func(*args, **kwargs) [ 755.882223] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 755.882223] env[64020]: raise e [ 755.882223] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 755.882223] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 755.882223] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 755.882223] env[64020]: created_port_ids = self._update_ports_for_instance( [ 755.882223] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 755.882223] env[64020]: with excutils.save_and_reraise_exception(): [ 755.882223] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 755.882223] env[64020]: self.force_reraise() [ 755.882223] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 755.882223] env[64020]: raise self.value [ 755.882223] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 755.882223] env[64020]: updated_port = self._update_port( [ 755.882223] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 755.882223] env[64020]: _ensure_no_port_binding_failure(port) [ 755.882223] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 755.882223] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 755.883076] env[64020]: nova.exception.PortBindingFailed: Binding failed for port c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8, please check neutron logs for more information. [ 755.883076] env[64020]: Removing descriptor: 18 [ 756.084081] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Releasing lock "refresh_cache-e01583c9-744c-47ea-b7cf-a46b046fc854" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 756.084522] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 756.084717] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 756.085023] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-64e444ac-70b2-45bb-b004-77c1d227b4f6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.093952] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8c3d672-c1a1-4159-af5a-76de7c1ae1e0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.114750] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e01583c9-744c-47ea-b7cf-a46b046fc854 could not be found. [ 756.114959] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 756.115133] env[64020]: INFO nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Took 0.03 seconds to destroy the instance on the hypervisor. [ 756.115368] env[64020]: DEBUG oslo.service.loopingcall [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 756.115580] env[64020]: DEBUG nova.compute.manager [-] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 756.115668] env[64020]: DEBUG nova.network.neutron [-] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 756.130341] env[64020]: DEBUG nova.network.neutron [-] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 756.130431] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0c7915fd56a845c28aa35d0159875a3c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 756.137296] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c7915fd56a845c28aa35d0159875a3c [ 756.179798] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.921s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 756.180395] env[64020]: ERROR nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9e393b6b-5a62-425c-98a6-2cae8538407d, please check neutron logs for more information. [ 756.180395] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Traceback (most recent call last): [ 756.180395] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 756.180395] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] self.driver.spawn(context, instance, image_meta, [ 756.180395] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 756.180395] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] self._vmops.spawn(context, instance, image_meta, injected_files, [ 756.180395] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 756.180395] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] vm_ref = self.build_virtual_machine(instance, [ 756.180395] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 756.180395] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] vif_infos = vmwarevif.get_vif_info(self._session, [ 756.180395] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] for vif in network_info: [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] return self._sync_wrapper(fn, *args, **kwargs) [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] self.wait() [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] self[:] = self._gt.wait() [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] return self._exit_event.wait() [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] result = hub.switch() [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 756.180738] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] return self.greenlet.switch() [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] result = function(*args, **kwargs) [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] return func(*args, **kwargs) [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] raise e [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] nwinfo = self.network_api.allocate_for_instance( [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] created_port_ids = self._update_ports_for_instance( [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] with excutils.save_and_reraise_exception(): [ 756.181171] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 756.181498] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] self.force_reraise() [ 756.181498] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 756.181498] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] raise self.value [ 756.181498] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 756.181498] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] updated_port = self._update_port( [ 756.181498] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 756.181498] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] _ensure_no_port_binding_failure(port) [ 756.181498] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 756.181498] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] raise exception.PortBindingFailed(port_id=port['id']) [ 756.181498] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] nova.exception.PortBindingFailed: Binding failed for port 9e393b6b-5a62-425c-98a6-2cae8538407d, please check neutron logs for more information. [ 756.181498] env[64020]: ERROR nova.compute.manager [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] [ 756.181773] env[64020]: DEBUG nova.compute.utils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Binding failed for port 9e393b6b-5a62-425c-98a6-2cae8538407d, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 756.182343] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 31.113s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 756.184179] env[64020]: INFO nova.compute.claims [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 756.186160] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 43537046ee0b437abccb3af6d074b13d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 756.187353] env[64020]: DEBUG nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Build of instance 5e43d3ea-740c-4569-be82-3bb7082fa8ea was re-scheduled: Binding failed for port 9e393b6b-5a62-425c-98a6-2cae8538407d, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 756.187794] env[64020]: DEBUG nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 756.188420] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Acquiring lock "refresh_cache-5e43d3ea-740c-4569-be82-3bb7082fa8ea" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 756.188420] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Acquired lock "refresh_cache-5e43d3ea-740c-4569-be82-3bb7082fa8ea" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 756.188420] env[64020]: DEBUG nova.network.neutron [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 756.188717] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg d11464628321427dbe559dfab75a797a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 756.198136] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d11464628321427dbe559dfab75a797a [ 756.225317] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43537046ee0b437abccb3af6d074b13d [ 756.278858] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 756.304947] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 756.305184] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 756.305322] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 756.305494] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 756.305627] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 756.305764] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 756.305956] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 756.306103] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 756.306269] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 756.306422] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 756.306581] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 756.307503] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94f32f2c-82da-4c79-9880-a30ea81ae76c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.315658] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ffbc955-0967-467c-a4e9-c9f4295cef01 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.329994] env[64020]: ERROR nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8, please check neutron logs for more information. [ 756.329994] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] Traceback (most recent call last): [ 756.329994] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 756.329994] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] yield resources [ 756.329994] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 756.329994] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] self.driver.spawn(context, instance, image_meta, [ 756.329994] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 756.329994] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] self._vmops.spawn(context, instance, image_meta, injected_files, [ 756.329994] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 756.329994] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] vm_ref = self.build_virtual_machine(instance, [ 756.329994] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] vif_infos = vmwarevif.get_vif_info(self._session, [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] for vif in network_info: [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] return self._sync_wrapper(fn, *args, **kwargs) [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] self.wait() [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] self[:] = self._gt.wait() [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] return self._exit_event.wait() [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 756.330382] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] current.throw(*self._exc) [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] result = function(*args, **kwargs) [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] return func(*args, **kwargs) [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] raise e [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] nwinfo = self.network_api.allocate_for_instance( [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] created_port_ids = self._update_ports_for_instance( [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] with excutils.save_and_reraise_exception(): [ 756.330841] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 756.331241] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] self.force_reraise() [ 756.331241] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 756.331241] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] raise self.value [ 756.331241] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 756.331241] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] updated_port = self._update_port( [ 756.331241] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 756.331241] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] _ensure_no_port_binding_failure(port) [ 756.331241] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 756.331241] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] raise exception.PortBindingFailed(port_id=port['id']) [ 756.331241] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] nova.exception.PortBindingFailed: Binding failed for port c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8, please check neutron logs for more information. [ 756.331241] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] [ 756.331241] env[64020]: INFO nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Terminating instance [ 756.332465] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "refresh_cache-786a572c-adaa-4656-8b42-c510940dac59" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 756.379075] env[64020]: DEBUG nova.network.neutron [req-d561604b-04d8-4a5c-ad64-4f21fede526d req-4015d35e-59c9-4268-b841-f9126d31f380 service nova] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 756.470877] env[64020]: DEBUG nova.network.neutron [req-d561604b-04d8-4a5c-ad64-4f21fede526d req-4015d35e-59c9-4268-b841-f9126d31f380 service nova] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 756.471492] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-d561604b-04d8-4a5c-ad64-4f21fede526d req-4015d35e-59c9-4268-b841-f9126d31f380 service nova] Expecting reply to msg 0afd1a761fec49b5a7c21309d66e808c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 756.479709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0afd1a761fec49b5a7c21309d66e808c [ 756.632601] env[64020]: DEBUG nova.network.neutron [-] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 756.632888] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c92decf119c0479bad904eb616ceae9d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 756.641578] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c92decf119c0479bad904eb616ceae9d [ 756.691953] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 6a4f943a8b3a43efa2cf7975bac1f9da in queue reply_57893177120949e6a93cb88e15cd42b4 [ 756.700159] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a4f943a8b3a43efa2cf7975bac1f9da [ 756.706953] env[64020]: DEBUG nova.network.neutron [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 756.772994] env[64020]: DEBUG nova.network.neutron [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 756.773459] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg df807a092fb54544837a85a1e94beb80 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 756.780697] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg df807a092fb54544837a85a1e94beb80 [ 756.974342] env[64020]: DEBUG oslo_concurrency.lockutils [req-d561604b-04d8-4a5c-ad64-4f21fede526d req-4015d35e-59c9-4268-b841-f9126d31f380 service nova] Releasing lock "refresh_cache-786a572c-adaa-4656-8b42-c510940dac59" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 756.974746] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquired lock "refresh_cache-786a572c-adaa-4656-8b42-c510940dac59" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 756.974922] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 756.975346] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 4fd0b707e80249f3a1208bbbdbde79d2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 756.982203] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4fd0b707e80249f3a1208bbbdbde79d2 [ 757.136037] env[64020]: INFO nova.compute.manager [-] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Took 1.02 seconds to deallocate network for instance. [ 757.138354] env[64020]: DEBUG nova.compute.claims [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 757.138450] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.275550] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Releasing lock "refresh_cache-5e43d3ea-740c-4569-be82-3bb7082fa8ea" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 757.275550] env[64020]: DEBUG nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 757.275696] env[64020]: DEBUG nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 757.275739] env[64020]: DEBUG nova.network.neutron [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 757.290373] env[64020]: DEBUG nova.network.neutron [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 757.290907] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 1aaf449d413e450dab4b18b52607dccf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 757.298203] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1aaf449d413e450dab4b18b52607dccf [ 757.493140] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 757.534973] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f14d69c7-1489-44b0-ad25-803418ec75ce {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.542839] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-befc2e5f-f99f-46cb-8f69-9200c28b9b03 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.575221] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 757.575742] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 6fc2ba7c5bdd427b961e3139512d4a0a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 757.582285] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d079e3c-adf8-48ba-abdd-263bb997a296 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.584890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6fc2ba7c5bdd427b961e3139512d4a0a [ 757.586174] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7d71329-6ec1-4ed6-a2d7-832417198226 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.600452] env[64020]: DEBUG nova.compute.provider_tree [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 757.600935] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg d9e0ca47f7574b72bbc51c74bc788b99 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 757.607606] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9e0ca47f7574b72bbc51c74bc788b99 [ 757.793930] env[64020]: DEBUG nova.network.neutron [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 757.794469] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 309bfcf1db134b728674a7af85189a98 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 757.802844] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 309bfcf1db134b728674a7af85189a98 [ 757.885567] env[64020]: DEBUG nova.compute.manager [req-dd9776ea-a97a-4638-84a7-0cb80ac11096 req-37aa6440-1e78-4d08-a225-d28307c7e988 service nova] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Received event network-vif-deleted-c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 758.081670] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Releasing lock "refresh_cache-786a572c-adaa-4656-8b42-c510940dac59" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 758.082076] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 758.082294] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 758.082586] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-18d42e44-4e77-48e0-960a-ed4211858708 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 758.091684] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c813d6c6-22c5-4e80-8d91-9b9670e11c92 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 758.103347] env[64020]: DEBUG nova.scheduler.client.report [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 758.105660] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 43d6f8319f8c4b86a452df22cc26e54f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 758.112528] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 786a572c-adaa-4656-8b42-c510940dac59 could not be found. [ 758.112699] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 758.112872] env[64020]: INFO nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Took 0.03 seconds to destroy the instance on the hypervisor. [ 758.113103] env[64020]: DEBUG oslo.service.loopingcall [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 758.113316] env[64020]: DEBUG nova.compute.manager [-] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 758.113410] env[64020]: DEBUG nova.network.neutron [-] [instance: 786a572c-adaa-4656-8b42-c510940dac59] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 758.119849] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43d6f8319f8c4b86a452df22cc26e54f [ 758.127442] env[64020]: DEBUG nova.network.neutron [-] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 758.127870] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ca2ca5294c044f10b4fdf4c832035c7c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 758.133908] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca2ca5294c044f10b4fdf4c832035c7c [ 758.296930] env[64020]: INFO nova.compute.manager [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] [instance: 5e43d3ea-740c-4569-be82-3bb7082fa8ea] Took 1.02 seconds to deallocate network for instance. [ 758.298762] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 03d94ec3dd4d4dc8a84298a2e6acda3f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 758.330403] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03d94ec3dd4d4dc8a84298a2e6acda3f [ 758.608296] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.426s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 758.608820] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 758.610535] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg e2dbc58d57164563b9d53f2b9f0d6ae9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 758.611561] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.158s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 758.613745] env[64020]: INFO nova.compute.claims [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 758.615361] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 91618e2ef978422ca2e846b95b645f9f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 758.629150] env[64020]: DEBUG nova.network.neutron [-] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 758.629526] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg fd7a39e88460472e951fd44282c56655 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 758.650354] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd7a39e88460472e951fd44282c56655 [ 758.654183] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91618e2ef978422ca2e846b95b645f9f [ 758.657316] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e2dbc58d57164563b9d53f2b9f0d6ae9 [ 758.803350] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 18671d90a5634cd48947548ef8409f69 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 758.835458] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18671d90a5634cd48947548ef8409f69 [ 759.119045] env[64020]: DEBUG nova.compute.utils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 759.119775] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 8a9e255232dd429fa711e487fc579164 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 759.121967] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 723a3961bb4a442faebf38d71a8fc120 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 759.122891] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 759.123069] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 759.131138] env[64020]: INFO nova.compute.manager [-] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Took 1.02 seconds to deallocate network for instance. [ 759.133155] env[64020]: DEBUG nova.compute.claims [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 759.133322] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 759.134083] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 723a3961bb4a442faebf38d71a8fc120 [ 759.134544] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a9e255232dd429fa711e487fc579164 [ 759.163552] env[64020]: DEBUG nova.policy [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '478fd14e8faf4a9e844bf53e9be91892', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5571feef12fc4ec6a09b15b69abeb4e5', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 759.323118] env[64020]: INFO nova.scheduler.client.report [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Deleted allocations for instance 5e43d3ea-740c-4569-be82-3bb7082fa8ea [ 759.329734] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Expecting reply to msg 9a256f8bec254a519f90be9116b64753 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 759.343970] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9a256f8bec254a519f90be9116b64753 [ 759.451769] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Successfully created port: 25741d42-2714-4498-9df6-2f8af144e18b {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 759.627722] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 759.629602] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 51a8dde51f69421ea7c804cda0a3906a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 759.667224] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51a8dde51f69421ea7c804cda0a3906a [ 759.831585] env[64020]: DEBUG oslo_concurrency.lockutils [None req-3c762823-9b61-4ce7-8d55-727655a735e5 tempest-ServersTestFqdnHostnames-118163914 tempest-ServersTestFqdnHostnames-118163914-project-member] Lock "5e43d3ea-740c-4569-be82-3bb7082fa8ea" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 134.404s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 759.832366] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg d99279ae60b64b7cad8b2081bba20208 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 759.848642] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d99279ae60b64b7cad8b2081bba20208 [ 760.031580] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c617cce-4be6-4be8-a30d-82d39e7620f4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.039452] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-887dcdcf-f6d9-4e2a-b63f-83b665e0f5f3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.068832] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4a43686-85a8-45ab-9e1b-58be1ff40a45 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.076228] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8426474b-8a4b-470b-b52f-3bac2b2f200e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.091799] env[64020]: DEBUG nova.compute.provider_tree [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 760.092434] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 5a502ab0077648adb89c91707243703a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 760.099868] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a502ab0077648adb89c91707243703a [ 760.131961] env[64020]: DEBUG nova.compute.manager [req-d5eaedc3-4226-4ae4-b5f6-556b9e7d2d3d req-d435e60f-0be3-482d-bc61-53edbc73334f service nova] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Received event network-changed-25741d42-2714-4498-9df6-2f8af144e18b {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 760.132065] env[64020]: DEBUG nova.compute.manager [req-d5eaedc3-4226-4ae4-b5f6-556b9e7d2d3d req-d435e60f-0be3-482d-bc61-53edbc73334f service nova] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Refreshing instance network info cache due to event network-changed-25741d42-2714-4498-9df6-2f8af144e18b. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 760.132393] env[64020]: DEBUG oslo_concurrency.lockutils [req-d5eaedc3-4226-4ae4-b5f6-556b9e7d2d3d req-d435e60f-0be3-482d-bc61-53edbc73334f service nova] Acquiring lock "refresh_cache-8e8e1b50-fcae-41c6-8f5a-241692140cda" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 760.132574] env[64020]: DEBUG oslo_concurrency.lockutils [req-d5eaedc3-4226-4ae4-b5f6-556b9e7d2d3d req-d435e60f-0be3-482d-bc61-53edbc73334f service nova] Acquired lock "refresh_cache-8e8e1b50-fcae-41c6-8f5a-241692140cda" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 760.132738] env[64020]: DEBUG nova.network.neutron [req-d5eaedc3-4226-4ae4-b5f6-556b9e7d2d3d req-d435e60f-0be3-482d-bc61-53edbc73334f service nova] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Refreshing network info cache for port 25741d42-2714-4498-9df6-2f8af144e18b {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 760.133143] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-d5eaedc3-4226-4ae4-b5f6-556b9e7d2d3d req-d435e60f-0be3-482d-bc61-53edbc73334f service nova] Expecting reply to msg 88fbfbe908fe456886892c8839668254 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 760.136893] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg c738548e2b4d4fe08c4084a593721d6b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 760.139661] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88fbfbe908fe456886892c8839668254 [ 760.172902] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c738548e2b4d4fe08c4084a593721d6b [ 760.309024] env[64020]: ERROR nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 25741d42-2714-4498-9df6-2f8af144e18b, please check neutron logs for more information. [ 760.309024] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 760.309024] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 760.309024] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 760.309024] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 760.309024] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 760.309024] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 760.309024] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 760.309024] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 760.309024] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 760.309024] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 760.309024] env[64020]: ERROR nova.compute.manager raise self.value [ 760.309024] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 760.309024] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 760.309024] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 760.309024] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 760.309572] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 760.309572] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 760.309572] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 25741d42-2714-4498-9df6-2f8af144e18b, please check neutron logs for more information. [ 760.309572] env[64020]: ERROR nova.compute.manager [ 760.309572] env[64020]: Traceback (most recent call last): [ 760.309572] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 760.309572] env[64020]: listener.cb(fileno) [ 760.309572] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 760.309572] env[64020]: result = function(*args, **kwargs) [ 760.309572] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 760.309572] env[64020]: return func(*args, **kwargs) [ 760.309572] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 760.309572] env[64020]: raise e [ 760.309572] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 760.309572] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 760.309572] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 760.309572] env[64020]: created_port_ids = self._update_ports_for_instance( [ 760.309572] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 760.309572] env[64020]: with excutils.save_and_reraise_exception(): [ 760.309572] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 760.309572] env[64020]: self.force_reraise() [ 760.309572] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 760.309572] env[64020]: raise self.value [ 760.309572] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 760.309572] env[64020]: updated_port = self._update_port( [ 760.309572] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 760.309572] env[64020]: _ensure_no_port_binding_failure(port) [ 760.309572] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 760.309572] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 760.310477] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 25741d42-2714-4498-9df6-2f8af144e18b, please check neutron logs for more information. [ 760.310477] env[64020]: Removing descriptor: 18 [ 760.335229] env[64020]: DEBUG nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 760.337087] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 8f9af0046b9f4d1b8c9476a9628ac832 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 760.368088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f9af0046b9f4d1b8c9476a9628ac832 [ 760.596935] env[64020]: DEBUG nova.scheduler.client.report [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 760.599412] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg a0c3a6796fad49999b1e023b2c30df6f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 760.611368] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a0c3a6796fad49999b1e023b2c30df6f [ 760.640669] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 760.650607] env[64020]: DEBUG nova.network.neutron [req-d5eaedc3-4226-4ae4-b5f6-556b9e7d2d3d req-d435e60f-0be3-482d-bc61-53edbc73334f service nova] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 760.663324] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 760.663641] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 760.663802] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 760.663983] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 760.664144] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 760.664497] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 760.664705] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 760.664856] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 760.665014] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 760.665357] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 760.665582] env[64020]: DEBUG nova.virt.hardware [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 760.666433] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-279cff44-0ae7-4132-95b5-4c0a1950cfb6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.676255] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6af200d0-37de-4823-b74f-927252720e23 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.691352] env[64020]: ERROR nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 25741d42-2714-4498-9df6-2f8af144e18b, please check neutron logs for more information. [ 760.691352] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Traceback (most recent call last): [ 760.691352] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 760.691352] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] yield resources [ 760.691352] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 760.691352] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] self.driver.spawn(context, instance, image_meta, [ 760.691352] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 760.691352] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] self._vmops.spawn(context, instance, image_meta, injected_files, [ 760.691352] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 760.691352] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] vm_ref = self.build_virtual_machine(instance, [ 760.691352] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] vif_infos = vmwarevif.get_vif_info(self._session, [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] for vif in network_info: [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] return self._sync_wrapper(fn, *args, **kwargs) [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] self.wait() [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] self[:] = self._gt.wait() [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] return self._exit_event.wait() [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 760.691741] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] current.throw(*self._exc) [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] result = function(*args, **kwargs) [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] return func(*args, **kwargs) [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] raise e [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] nwinfo = self.network_api.allocate_for_instance( [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] created_port_ids = self._update_ports_for_instance( [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] with excutils.save_and_reraise_exception(): [ 760.692148] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 760.692638] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] self.force_reraise() [ 760.692638] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 760.692638] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] raise self.value [ 760.692638] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 760.692638] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] updated_port = self._update_port( [ 760.692638] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 760.692638] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] _ensure_no_port_binding_failure(port) [ 760.692638] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 760.692638] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] raise exception.PortBindingFailed(port_id=port['id']) [ 760.692638] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] nova.exception.PortBindingFailed: Binding failed for port 25741d42-2714-4498-9df6-2f8af144e18b, please check neutron logs for more information. [ 760.692638] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] [ 760.692638] env[64020]: INFO nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Terminating instance [ 760.695464] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "refresh_cache-8e8e1b50-fcae-41c6-8f5a-241692140cda" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 760.740202] env[64020]: DEBUG nova.network.neutron [req-d5eaedc3-4226-4ae4-b5f6-556b9e7d2d3d req-d435e60f-0be3-482d-bc61-53edbc73334f service nova] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 760.740709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-d5eaedc3-4226-4ae4-b5f6-556b9e7d2d3d req-d435e60f-0be3-482d-bc61-53edbc73334f service nova] Expecting reply to msg 73559609b6d24947816406a1d3e55d9f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 760.751078] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 73559609b6d24947816406a1d3e55d9f [ 760.858962] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.102836] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.491s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.103376] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 761.105110] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 1c6cf29e5029459aa43808b6effc1e8e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 761.106158] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.702s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.107557] env[64020]: INFO nova.compute.claims [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 761.109089] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 8cfd068f17ef4da4bdcd987f5524c475 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 761.165117] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8cfd068f17ef4da4bdcd987f5524c475 [ 761.173079] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1c6cf29e5029459aa43808b6effc1e8e [ 761.243278] env[64020]: DEBUG oslo_concurrency.lockutils [req-d5eaedc3-4226-4ae4-b5f6-556b9e7d2d3d req-d435e60f-0be3-482d-bc61-53edbc73334f service nova] Releasing lock "refresh_cache-8e8e1b50-fcae-41c6-8f5a-241692140cda" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 761.243913] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquired lock "refresh_cache-8e8e1b50-fcae-41c6-8f5a-241692140cda" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 761.244243] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 761.244893] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg e3fb712adc244b33a9c6129d320832d1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 761.255283] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3fb712adc244b33a9c6129d320832d1 [ 761.612925] env[64020]: DEBUG nova.compute.utils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 761.613902] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 5a12cc74d13a48558428f82081ae6813 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 761.616731] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg e8d7738f5d584588b60e9e529636a733 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 761.618433] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 761.618643] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 761.625416] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e8d7738f5d584588b60e9e529636a733 [ 761.627581] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a12cc74d13a48558428f82081ae6813 [ 761.688150] env[64020]: DEBUG nova.policy [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6eef8429eed24f899b44e5e605c70791', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a6629d1c8cd94df7a442e558e0ff4f08', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 761.778006] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 761.883118] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 761.883661] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 8cb3d6e4e4cc4758941f87bda1645cf8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 761.892027] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8cb3d6e4e4cc4758941f87bda1645cf8 [ 762.119206] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 762.120982] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 5bb827d7e1ed48618b532a6be3049db4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 762.158844] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Successfully created port: 1558d2ac-f57e-47e6-81d9-e75b57f46aee {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 762.162863] env[64020]: DEBUG nova.compute.manager [req-8224700d-8cc3-4cb4-a5c0-1017550ed02f req-aab88336-7b5b-47d7-8f27-58fba84e1834 service nova] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Received event network-vif-deleted-25741d42-2714-4498-9df6-2f8af144e18b {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 762.163499] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5bb827d7e1ed48618b532a6be3049db4 [ 762.387238] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Releasing lock "refresh_cache-8e8e1b50-fcae-41c6-8f5a-241692140cda" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 762.387647] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 762.387837] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 762.388321] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9fdc775c-9343-4ef9-831f-4c69a6c7497e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.396556] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7a5ba34-82bb-470c-bd94-33f166634732 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.426323] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 8e8e1b50-fcae-41c6-8f5a-241692140cda could not be found. [ 762.426699] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 762.426875] env[64020]: INFO nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Took 0.04 seconds to destroy the instance on the hypervisor. [ 762.427145] env[64020]: DEBUG oslo.service.loopingcall [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 762.427397] env[64020]: DEBUG nova.compute.manager [-] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 762.427497] env[64020]: DEBUG nova.network.neutron [-] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 762.471466] env[64020]: DEBUG nova.network.neutron [-] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 762.472072] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 9219394313484602b1ebcfd14bab86ca in queue reply_57893177120949e6a93cb88e15cd42b4 [ 762.479300] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9219394313484602b1ebcfd14bab86ca [ 762.510047] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f620eb7f-1cfe-483c-9b8b-c457f009043f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.517274] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee5dd5e0-552e-4513-b983-b20eb780473f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.553926] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7fdbba78-63ac-468a-809d-cede93bf4072 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.561243] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e88117d-9a2a-4b58-b003-85bae44ca659 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.575864] env[64020]: DEBUG nova.compute.provider_tree [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 762.576397] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 52ffd7bb3ee64870bc9205a663e8a324 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 762.586216] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52ffd7bb3ee64870bc9205a663e8a324 [ 762.629922] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg b194fd0e55e845eeb6a0bbd75bec6835 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 762.673395] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b194fd0e55e845eeb6a0bbd75bec6835 [ 762.974224] env[64020]: DEBUG nova.network.neutron [-] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 762.974744] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0daf4f1b042e4bf490eecbf722364e11 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 762.984676] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0daf4f1b042e4bf490eecbf722364e11 [ 763.081098] env[64020]: DEBUG nova.scheduler.client.report [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 763.083402] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg f614f85d12b94cec9e5076ad90ec7edf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 763.095296] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f614f85d12b94cec9e5076ad90ec7edf [ 763.134646] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 763.164443] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 763.164707] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 763.164861] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 763.165041] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 763.165184] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 763.165328] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 763.165532] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 763.165686] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 763.165839] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 763.165990] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 763.166168] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 763.167341] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6dc70144-60d7-44f5-82fc-4cf199b8a9fe {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 763.175250] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1eb3c04a-5e17-422a-a6a6-44ee5eb7ca3d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 763.207902] env[64020]: ERROR nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1558d2ac-f57e-47e6-81d9-e75b57f46aee, please check neutron logs for more information. [ 763.207902] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 763.207902] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 763.207902] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 763.207902] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 763.207902] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 763.207902] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 763.207902] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 763.207902] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 763.207902] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 763.207902] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 763.207902] env[64020]: ERROR nova.compute.manager raise self.value [ 763.207902] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 763.207902] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 763.207902] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 763.207902] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 763.208418] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 763.208418] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 763.208418] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1558d2ac-f57e-47e6-81d9-e75b57f46aee, please check neutron logs for more information. [ 763.208418] env[64020]: ERROR nova.compute.manager [ 763.208418] env[64020]: Traceback (most recent call last): [ 763.208418] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 763.208418] env[64020]: listener.cb(fileno) [ 763.208418] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 763.208418] env[64020]: result = function(*args, **kwargs) [ 763.208418] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 763.208418] env[64020]: return func(*args, **kwargs) [ 763.208418] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 763.208418] env[64020]: raise e [ 763.208418] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 763.208418] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 763.208418] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 763.208418] env[64020]: created_port_ids = self._update_ports_for_instance( [ 763.208418] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 763.208418] env[64020]: with excutils.save_and_reraise_exception(): [ 763.208418] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 763.208418] env[64020]: self.force_reraise() [ 763.208418] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 763.208418] env[64020]: raise self.value [ 763.208418] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 763.208418] env[64020]: updated_port = self._update_port( [ 763.208418] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 763.208418] env[64020]: _ensure_no_port_binding_failure(port) [ 763.208418] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 763.208418] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 763.209377] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 1558d2ac-f57e-47e6-81d9-e75b57f46aee, please check neutron logs for more information. [ 763.209377] env[64020]: Removing descriptor: 18 [ 763.209377] env[64020]: ERROR nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1558d2ac-f57e-47e6-81d9-e75b57f46aee, please check neutron logs for more information. [ 763.209377] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Traceback (most recent call last): [ 763.209377] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 763.209377] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] yield resources [ 763.209377] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 763.209377] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] self.driver.spawn(context, instance, image_meta, [ 763.209377] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 763.209377] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] self._vmops.spawn(context, instance, image_meta, injected_files, [ 763.209377] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 763.209377] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] vm_ref = self.build_virtual_machine(instance, [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] vif_infos = vmwarevif.get_vif_info(self._session, [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] for vif in network_info: [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] return self._sync_wrapper(fn, *args, **kwargs) [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] self.wait() [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] self[:] = self._gt.wait() [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] return self._exit_event.wait() [ 763.209824] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] result = hub.switch() [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] return self.greenlet.switch() [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] result = function(*args, **kwargs) [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] return func(*args, **kwargs) [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] raise e [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] nwinfo = self.network_api.allocate_for_instance( [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 763.210292] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] created_port_ids = self._update_ports_for_instance( [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] with excutils.save_and_reraise_exception(): [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] self.force_reraise() [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] raise self.value [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] updated_port = self._update_port( [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] _ensure_no_port_binding_failure(port) [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 763.210731] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] raise exception.PortBindingFailed(port_id=port['id']) [ 763.211343] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] nova.exception.PortBindingFailed: Binding failed for port 1558d2ac-f57e-47e6-81d9-e75b57f46aee, please check neutron logs for more information. [ 763.211343] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] [ 763.211343] env[64020]: INFO nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Terminating instance [ 763.211343] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "refresh_cache-d808804b-a2d0-4225-beb5-7d91e0cff3ac" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 763.211575] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquired lock "refresh_cache-d808804b-a2d0-4225-beb5-7d91e0cff3ac" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 763.211575] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 763.211975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 6508f42698cf489eae73d9081dc4202d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 763.218189] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6508f42698cf489eae73d9081dc4202d [ 763.477075] env[64020]: INFO nova.compute.manager [-] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Took 1.05 seconds to deallocate network for instance. [ 763.479529] env[64020]: DEBUG nova.compute.claims [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 763.479690] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 763.585797] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.480s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 763.586425] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 763.588050] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 5bbb0afa7a974a8ab5440804aa499a2b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 763.589011] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.172s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 763.590650] env[64020]: INFO nova.compute.claims [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 763.592487] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 5738788bfc91410fa2dc8347bfba12b8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 763.628697] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5bbb0afa7a974a8ab5440804aa499a2b [ 763.629329] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5738788bfc91410fa2dc8347bfba12b8 [ 763.728441] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 763.808420] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 763.808838] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 61c07ebfa86a43d386fa67b0adf8a240 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 763.818321] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 61c07ebfa86a43d386fa67b0adf8a240 [ 764.096976] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 6ad92545b33e470c9bae1c17dea078af in queue reply_57893177120949e6a93cb88e15cd42b4 [ 764.098591] env[64020]: DEBUG nova.compute.utils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 764.099157] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 2d75082316fd4b87ba364ee2292874ad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 764.100202] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 764.100375] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 764.106947] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ad92545b33e470c9bae1c17dea078af [ 764.108456] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d75082316fd4b87ba364ee2292874ad [ 764.144183] env[64020]: DEBUG nova.policy [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6eef8429eed24f899b44e5e605c70791', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a6629d1c8cd94df7a442e558e0ff4f08', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 764.203353] env[64020]: DEBUG nova.compute.manager [req-72fa805e-919b-4fda-bac2-773334338544 req-0ade9140-8b38-4418-8a8a-9fcb2d2841ab service nova] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Received event network-changed-1558d2ac-f57e-47e6-81d9-e75b57f46aee {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 764.203536] env[64020]: DEBUG nova.compute.manager [req-72fa805e-919b-4fda-bac2-773334338544 req-0ade9140-8b38-4418-8a8a-9fcb2d2841ab service nova] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Refreshing instance network info cache due to event network-changed-1558d2ac-f57e-47e6-81d9-e75b57f46aee. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 764.203721] env[64020]: DEBUG oslo_concurrency.lockutils [req-72fa805e-919b-4fda-bac2-773334338544 req-0ade9140-8b38-4418-8a8a-9fcb2d2841ab service nova] Acquiring lock "refresh_cache-d808804b-a2d0-4225-beb5-7d91e0cff3ac" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 764.311140] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Releasing lock "refresh_cache-d808804b-a2d0-4225-beb5-7d91e0cff3ac" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 764.311601] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 764.311797] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 764.312144] env[64020]: DEBUG oslo_concurrency.lockutils [req-72fa805e-919b-4fda-bac2-773334338544 req-0ade9140-8b38-4418-8a8a-9fcb2d2841ab service nova] Acquired lock "refresh_cache-d808804b-a2d0-4225-beb5-7d91e0cff3ac" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 764.312315] env[64020]: DEBUG nova.network.neutron [req-72fa805e-919b-4fda-bac2-773334338544 req-0ade9140-8b38-4418-8a8a-9fcb2d2841ab service nova] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Refreshing network info cache for port 1558d2ac-f57e-47e6-81d9-e75b57f46aee {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 764.312756] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-72fa805e-919b-4fda-bac2-773334338544 req-0ade9140-8b38-4418-8a8a-9fcb2d2841ab service nova] Expecting reply to msg 5e1026278ad44e6594bfc114207e4b61 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 764.313905] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ea5934b7-d477-44df-b160-750b6cee3a92 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.320407] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e1026278ad44e6594bfc114207e4b61 [ 764.323349] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8de1d648-8834-4e60-a2bc-b4b30cba204e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.346051] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d808804b-a2d0-4225-beb5-7d91e0cff3ac could not be found. [ 764.346313] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 764.346504] env[64020]: INFO nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Took 0.03 seconds to destroy the instance on the hypervisor. [ 764.346743] env[64020]: DEBUG oslo.service.loopingcall [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 764.346954] env[64020]: DEBUG nova.compute.manager [-] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 764.347048] env[64020]: DEBUG nova.network.neutron [-] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 764.370800] env[64020]: DEBUG nova.network.neutron [-] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 764.370971] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5646b589f9bf42789b9d27d2def6db58 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 764.378049] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5646b589f9bf42789b9d27d2def6db58 [ 764.554244] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Successfully created port: fb363812-d8d1-49c5-95b7-693cacafc863 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 764.603720] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 764.605411] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg bf4936eace9a49068f5f2e1b25266c79 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 764.640062] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf4936eace9a49068f5f2e1b25266c79 [ 764.843032] env[64020]: DEBUG nova.network.neutron [req-72fa805e-919b-4fda-bac2-773334338544 req-0ade9140-8b38-4418-8a8a-9fcb2d2841ab service nova] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 764.873922] env[64020]: DEBUG nova.network.neutron [-] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 764.874361] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b6ff366a9c7d4cb086b994e11e20d9f8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 764.883598] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b6ff366a9c7d4cb086b994e11e20d9f8 [ 764.957349] env[64020]: DEBUG nova.network.neutron [req-72fa805e-919b-4fda-bac2-773334338544 req-0ade9140-8b38-4418-8a8a-9fcb2d2841ab service nova] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 764.957568] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-72fa805e-919b-4fda-bac2-773334338544 req-0ade9140-8b38-4418-8a8a-9fcb2d2841ab service nova] Expecting reply to msg a2ce6751ee4f4e8f80ecb38cdc0e0fbb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 764.966874] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2ce6751ee4f4e8f80ecb38cdc0e0fbb [ 764.991460] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d6fe6f9-dac2-43b4-9c1d-9f89c5702862 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.000143] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35bdaf2c-1768-426d-a2ae-7a2d339f59f4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.030864] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ebfb81d-ee94-4076-b4ec-d809b57ce18d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.038099] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d48cd8c-fa35-400b-be60-7efc984f6cd1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.051820] env[64020]: DEBUG nova.compute.provider_tree [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 765.052350] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 423af273ff344850b9a2a3fdbab3773e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 765.065477] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 423af273ff344850b9a2a3fdbab3773e [ 765.110342] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 3e4b8d9d03be4de39376159db59b882c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 765.144623] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e4b8d9d03be4de39376159db59b882c [ 765.376511] env[64020]: INFO nova.compute.manager [-] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Took 1.03 seconds to deallocate network for instance. [ 765.378844] env[64020]: DEBUG nova.compute.claims [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 765.379158] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 765.427812] env[64020]: ERROR nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port fb363812-d8d1-49c5-95b7-693cacafc863, please check neutron logs for more information. [ 765.427812] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 765.427812] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 765.427812] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 765.427812] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 765.427812] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 765.427812] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 765.427812] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 765.427812] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 765.427812] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 765.427812] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 765.427812] env[64020]: ERROR nova.compute.manager raise self.value [ 765.427812] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 765.427812] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 765.427812] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 765.427812] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 765.428364] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 765.428364] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 765.428364] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port fb363812-d8d1-49c5-95b7-693cacafc863, please check neutron logs for more information. [ 765.428364] env[64020]: ERROR nova.compute.manager [ 765.428364] env[64020]: Traceback (most recent call last): [ 765.428364] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 765.428364] env[64020]: listener.cb(fileno) [ 765.428364] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 765.428364] env[64020]: result = function(*args, **kwargs) [ 765.428364] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 765.428364] env[64020]: return func(*args, **kwargs) [ 765.428364] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 765.428364] env[64020]: raise e [ 765.428364] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 765.428364] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 765.428364] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 765.428364] env[64020]: created_port_ids = self._update_ports_for_instance( [ 765.428364] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 765.428364] env[64020]: with excutils.save_and_reraise_exception(): [ 765.428364] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 765.428364] env[64020]: self.force_reraise() [ 765.428364] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 765.428364] env[64020]: raise self.value [ 765.428364] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 765.428364] env[64020]: updated_port = self._update_port( [ 765.428364] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 765.428364] env[64020]: _ensure_no_port_binding_failure(port) [ 765.428364] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 765.428364] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 765.429252] env[64020]: nova.exception.PortBindingFailed: Binding failed for port fb363812-d8d1-49c5-95b7-693cacafc863, please check neutron logs for more information. [ 765.429252] env[64020]: Removing descriptor: 18 [ 765.460781] env[64020]: DEBUG oslo_concurrency.lockutils [req-72fa805e-919b-4fda-bac2-773334338544 req-0ade9140-8b38-4418-8a8a-9fcb2d2841ab service nova] Releasing lock "refresh_cache-d808804b-a2d0-4225-beb5-7d91e0cff3ac" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 765.461049] env[64020]: DEBUG nova.compute.manager [req-72fa805e-919b-4fda-bac2-773334338544 req-0ade9140-8b38-4418-8a8a-9fcb2d2841ab service nova] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Received event network-vif-deleted-1558d2ac-f57e-47e6-81d9-e75b57f46aee {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 765.556070] env[64020]: DEBUG nova.scheduler.client.report [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 765.559208] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg d61b540b3b914d6aacf9a8ba11bc5e09 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 765.588130] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d61b540b3b914d6aacf9a8ba11bc5e09 [ 765.612987] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 765.639296] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 765.639633] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 765.639881] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 765.640209] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 765.640419] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 765.640627] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 765.640908] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 765.641177] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 765.641384] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 765.641611] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 765.641838] env[64020]: DEBUG nova.virt.hardware [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 765.642773] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f89eece-9a77-4421-a6a2-18687e725e5a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.650969] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10d3b29d-ffad-4627-91e6-2d3dd2fdc545 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.665535] env[64020]: ERROR nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port fb363812-d8d1-49c5-95b7-693cacafc863, please check neutron logs for more information. [ 765.665535] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Traceback (most recent call last): [ 765.665535] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 765.665535] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] yield resources [ 765.665535] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 765.665535] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] self.driver.spawn(context, instance, image_meta, [ 765.665535] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 765.665535] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 765.665535] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 765.665535] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] vm_ref = self.build_virtual_machine(instance, [ 765.665535] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] vif_infos = vmwarevif.get_vif_info(self._session, [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] for vif in network_info: [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] return self._sync_wrapper(fn, *args, **kwargs) [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] self.wait() [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] self[:] = self._gt.wait() [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] return self._exit_event.wait() [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 765.665939] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] current.throw(*self._exc) [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] result = function(*args, **kwargs) [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] return func(*args, **kwargs) [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] raise e [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] nwinfo = self.network_api.allocate_for_instance( [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] created_port_ids = self._update_ports_for_instance( [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] with excutils.save_and_reraise_exception(): [ 765.666368] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 765.666728] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] self.force_reraise() [ 765.666728] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 765.666728] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] raise self.value [ 765.666728] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 765.666728] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] updated_port = self._update_port( [ 765.666728] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 765.666728] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] _ensure_no_port_binding_failure(port) [ 765.666728] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 765.666728] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] raise exception.PortBindingFailed(port_id=port['id']) [ 765.666728] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] nova.exception.PortBindingFailed: Binding failed for port fb363812-d8d1-49c5-95b7-693cacafc863, please check neutron logs for more information. [ 765.666728] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] [ 765.666728] env[64020]: INFO nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Terminating instance [ 765.669319] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "refresh_cache-2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 765.669544] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquired lock "refresh_cache-2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 765.669758] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 765.670219] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 38ba71362844407d871338ceee6231e2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 765.677028] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 38ba71362844407d871338ceee6231e2 [ 766.073808] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.485s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 766.074540] env[64020]: DEBUG nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 766.076136] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg bf7ede7a01c84cf3a56f6f04dc440a7f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 766.077543] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.120s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 766.079218] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 3181bd2b09e9410c93470f29c8407847 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 766.112071] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf7ede7a01c84cf3a56f6f04dc440a7f [ 766.112677] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3181bd2b09e9410c93470f29c8407847 [ 766.187047] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 766.232854] env[64020]: DEBUG nova.compute.manager [req-915cac3d-4fb4-4419-8abb-6b26cc9d5b4f req-560c1817-6af5-42e9-8311-ee0c3240b578 service nova] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Received event network-changed-fb363812-d8d1-49c5-95b7-693cacafc863 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 766.233050] env[64020]: DEBUG nova.compute.manager [req-915cac3d-4fb4-4419-8abb-6b26cc9d5b4f req-560c1817-6af5-42e9-8311-ee0c3240b578 service nova] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Refreshing instance network info cache due to event network-changed-fb363812-d8d1-49c5-95b7-693cacafc863. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 766.233242] env[64020]: DEBUG oslo_concurrency.lockutils [req-915cac3d-4fb4-4419-8abb-6b26cc9d5b4f req-560c1817-6af5-42e9-8311-ee0c3240b578 service nova] Acquiring lock "refresh_cache-2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 766.291336] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 766.291336] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 030d282bbf14492d95aad87348d181ea in queue reply_57893177120949e6a93cb88e15cd42b4 [ 766.299667] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 030d282bbf14492d95aad87348d181ea [ 766.587163] env[64020]: DEBUG nova.compute.utils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 766.587163] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 474fe05003944896a1e18eee15e52d8e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 766.587689] env[64020]: DEBUG nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 766.588203] env[64020]: DEBUG nova.network.neutron [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 766.603289] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 474fe05003944896a1e18eee15e52d8e [ 766.642252] env[64020]: DEBUG nova.policy [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '354ed73a9fce490098c4c3fe236a6ff9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c6789552a001499dbbac7aaf722cfdf3', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 766.792647] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Releasing lock "refresh_cache-2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 766.793064] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 766.793256] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 766.793754] env[64020]: DEBUG oslo_concurrency.lockutils [req-915cac3d-4fb4-4419-8abb-6b26cc9d5b4f req-560c1817-6af5-42e9-8311-ee0c3240b578 service nova] Acquired lock "refresh_cache-2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 766.793926] env[64020]: DEBUG nova.network.neutron [req-915cac3d-4fb4-4419-8abb-6b26cc9d5b4f req-560c1817-6af5-42e9-8311-ee0c3240b578 service nova] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Refreshing network info cache for port fb363812-d8d1-49c5-95b7-693cacafc863 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 766.794349] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-915cac3d-4fb4-4419-8abb-6b26cc9d5b4f req-560c1817-6af5-42e9-8311-ee0c3240b578 service nova] Expecting reply to msg 95e3a313040040a7b101b5e0e9543617 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 766.800872] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3873e445-4f0d-4995-aaae-548a038e3050 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.812211] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95e3a313040040a7b101b5e0e9543617 [ 766.822964] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ca2e919-24df-4601-b575-8cf8aec72927 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.847208] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4 could not be found. [ 766.847488] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 766.847675] env[64020]: INFO nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Took 0.05 seconds to destroy the instance on the hypervisor. [ 766.847919] env[64020]: DEBUG oslo.service.loopingcall [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 766.850456] env[64020]: DEBUG nova.compute.manager [-] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 766.850553] env[64020]: DEBUG nova.network.neutron [-] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 766.865517] env[64020]: DEBUG nova.network.neutron [-] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 766.865985] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 01ca5f4bf7834fd4a5068f0aec44a73c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 766.873038] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 01ca5f4bf7834fd4a5068f0aec44a73c [ 766.961476] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68316212-328c-4dc6-9426-5fe5933bbd38 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.968924] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-177d49f1-5584-44dd-aedf-7e83f62c80c7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.998242] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8bf3d43-1d19-49c3-a90d-0e7564b1fec3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.001529] env[64020]: DEBUG nova.network.neutron [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Successfully created port: 260fff8f-aefb-4420-a11a-81a330ebc23c {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 767.008111] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a3f6424-01a7-4cad-af61-b2fb3307b11e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.021387] env[64020]: DEBUG nova.compute.provider_tree [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 767.021893] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 43b3533d226f497188cefb73acf22753 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 767.029399] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43b3533d226f497188cefb73acf22753 [ 767.091810] env[64020]: DEBUG nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 767.093883] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 6311abe31da24bc9bac9aae5f4620d5a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 767.130147] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6311abe31da24bc9bac9aae5f4620d5a [ 767.316410] env[64020]: DEBUG nova.network.neutron [req-915cac3d-4fb4-4419-8abb-6b26cc9d5b4f req-560c1817-6af5-42e9-8311-ee0c3240b578 service nova] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 767.369436] env[64020]: DEBUG nova.network.neutron [-] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 767.370044] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6cbabff520d1403797ba9667f0941657 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 767.384745] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6cbabff520d1403797ba9667f0941657 [ 767.424438] env[64020]: DEBUG nova.network.neutron [req-915cac3d-4fb4-4419-8abb-6b26cc9d5b4f req-560c1817-6af5-42e9-8311-ee0c3240b578 service nova] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 767.424935] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-915cac3d-4fb4-4419-8abb-6b26cc9d5b4f req-560c1817-6af5-42e9-8311-ee0c3240b578 service nova] Expecting reply to msg 802d3e68ea4a4214940a1275cf40c3f4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 767.433594] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 802d3e68ea4a4214940a1275cf40c3f4 [ 767.524202] env[64020]: DEBUG nova.scheduler.client.report [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 767.526627] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 4cd59e5bf5f84010ac525e14e4c6c510 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 767.538231] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4cd59e5bf5f84010ac525e14e4c6c510 [ 767.600239] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 04ebc76229de4c188b519f704ee243ba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 767.643521] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 04ebc76229de4c188b519f704ee243ba [ 767.795069] env[64020]: ERROR nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 260fff8f-aefb-4420-a11a-81a330ebc23c, please check neutron logs for more information. [ 767.795069] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 767.795069] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 767.795069] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 767.795069] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 767.795069] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 767.795069] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 767.795069] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 767.795069] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 767.795069] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 767.795069] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 767.795069] env[64020]: ERROR nova.compute.manager raise self.value [ 767.795069] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 767.795069] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 767.795069] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 767.795069] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 767.795576] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 767.795576] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 767.795576] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 260fff8f-aefb-4420-a11a-81a330ebc23c, please check neutron logs for more information. [ 767.795576] env[64020]: ERROR nova.compute.manager [ 767.795576] env[64020]: Traceback (most recent call last): [ 767.795576] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 767.795576] env[64020]: listener.cb(fileno) [ 767.795576] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 767.795576] env[64020]: result = function(*args, **kwargs) [ 767.795576] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 767.795576] env[64020]: return func(*args, **kwargs) [ 767.795576] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 767.795576] env[64020]: raise e [ 767.795576] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 767.795576] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 767.795576] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 767.795576] env[64020]: created_port_ids = self._update_ports_for_instance( [ 767.795576] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 767.795576] env[64020]: with excutils.save_and_reraise_exception(): [ 767.795576] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 767.795576] env[64020]: self.force_reraise() [ 767.795576] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 767.795576] env[64020]: raise self.value [ 767.795576] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 767.795576] env[64020]: updated_port = self._update_port( [ 767.795576] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 767.795576] env[64020]: _ensure_no_port_binding_failure(port) [ 767.795576] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 767.795576] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 767.796441] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 260fff8f-aefb-4420-a11a-81a330ebc23c, please check neutron logs for more information. [ 767.796441] env[64020]: Removing descriptor: 18 [ 767.871971] env[64020]: INFO nova.compute.manager [-] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Took 1.02 seconds to deallocate network for instance. [ 767.874443] env[64020]: DEBUG nova.compute.claims [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 767.874611] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.927840] env[64020]: DEBUG oslo_concurrency.lockutils [req-915cac3d-4fb4-4419-8abb-6b26cc9d5b4f req-560c1817-6af5-42e9-8311-ee0c3240b578 service nova] Releasing lock "refresh_cache-2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 767.928139] env[64020]: DEBUG nova.compute.manager [req-915cac3d-4fb4-4419-8abb-6b26cc9d5b4f req-560c1817-6af5-42e9-8311-ee0c3240b578 service nova] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Received event network-vif-deleted-fb363812-d8d1-49c5-95b7-693cacafc863 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 768.029331] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.952s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 768.029960] env[64020]: ERROR nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 991f8f50-f08a-4658-a355-5e0a04dc614f, please check neutron logs for more information. [ 768.029960] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Traceback (most recent call last): [ 768.029960] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 768.029960] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] self.driver.spawn(context, instance, image_meta, [ 768.029960] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 768.029960] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 768.029960] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 768.029960] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] vm_ref = self.build_virtual_machine(instance, [ 768.029960] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 768.029960] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] vif_infos = vmwarevif.get_vif_info(self._session, [ 768.029960] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] for vif in network_info: [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] return self._sync_wrapper(fn, *args, **kwargs) [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] self.wait() [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] self[:] = self._gt.wait() [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] return self._exit_event.wait() [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] current.throw(*self._exc) [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 768.030293] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] result = function(*args, **kwargs) [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] return func(*args, **kwargs) [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] raise e [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] nwinfo = self.network_api.allocate_for_instance( [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] created_port_ids = self._update_ports_for_instance( [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] with excutils.save_and_reraise_exception(): [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] self.force_reraise() [ 768.030623] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 768.030954] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] raise self.value [ 768.030954] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 768.030954] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] updated_port = self._update_port( [ 768.030954] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 768.030954] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] _ensure_no_port_binding_failure(port) [ 768.030954] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 768.030954] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] raise exception.PortBindingFailed(port_id=port['id']) [ 768.030954] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] nova.exception.PortBindingFailed: Binding failed for port 991f8f50-f08a-4658-a355-5e0a04dc614f, please check neutron logs for more information. [ 768.030954] env[64020]: ERROR nova.compute.manager [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] [ 768.030954] env[64020]: DEBUG nova.compute.utils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Binding failed for port 991f8f50-f08a-4658-a355-5e0a04dc614f, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 768.031867] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.940s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 768.033715] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg 75d2e92edbc547d484f36af54de584f2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 768.034883] env[64020]: DEBUG nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Build of instance 866f8ba4-faad-4883-80ca-4c4dedb2b50c was re-scheduled: Binding failed for port 991f8f50-f08a-4658-a355-5e0a04dc614f, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 768.035286] env[64020]: DEBUG nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 768.035493] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquiring lock "refresh_cache-866f8ba4-faad-4883-80ca-4c4dedb2b50c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 768.035630] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Acquired lock "refresh_cache-866f8ba4-faad-4883-80ca-4c4dedb2b50c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 768.035781] env[64020]: DEBUG nova.network.neutron [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 768.036150] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg add538ba1ce145b2b8d601f10d30ccc1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 768.060735] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg add538ba1ce145b2b8d601f10d30ccc1 [ 768.069817] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 75d2e92edbc547d484f36af54de584f2 [ 768.104705] env[64020]: DEBUG nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 768.127862] env[64020]: DEBUG nova.virt.hardware [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 768.128112] env[64020]: DEBUG nova.virt.hardware [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 768.128269] env[64020]: DEBUG nova.virt.hardware [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 768.128469] env[64020]: DEBUG nova.virt.hardware [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 768.128621] env[64020]: DEBUG nova.virt.hardware [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 768.128769] env[64020]: DEBUG nova.virt.hardware [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 768.128963] env[64020]: DEBUG nova.virt.hardware [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 768.129205] env[64020]: DEBUG nova.virt.hardware [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 768.129395] env[64020]: DEBUG nova.virt.hardware [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 768.129556] env[64020]: DEBUG nova.virt.hardware [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 768.129722] env[64020]: DEBUG nova.virt.hardware [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 768.130537] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2546bee-fc1f-4c4d-83ca-4d2077377243 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.138257] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd15a1f9-0fb8-48ef-840d-3b265ab850f7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.152711] env[64020]: ERROR nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 260fff8f-aefb-4420-a11a-81a330ebc23c, please check neutron logs for more information. [ 768.152711] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Traceback (most recent call last): [ 768.152711] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 768.152711] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] yield resources [ 768.152711] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 768.152711] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] self.driver.spawn(context, instance, image_meta, [ 768.152711] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 768.152711] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] self._vmops.spawn(context, instance, image_meta, injected_files, [ 768.152711] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 768.152711] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] vm_ref = self.build_virtual_machine(instance, [ 768.152711] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] vif_infos = vmwarevif.get_vif_info(self._session, [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] for vif in network_info: [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] return self._sync_wrapper(fn, *args, **kwargs) [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] self.wait() [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] self[:] = self._gt.wait() [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] return self._exit_event.wait() [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 768.153064] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] current.throw(*self._exc) [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] result = function(*args, **kwargs) [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] return func(*args, **kwargs) [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] raise e [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] nwinfo = self.network_api.allocate_for_instance( [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] created_port_ids = self._update_ports_for_instance( [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] with excutils.save_and_reraise_exception(): [ 768.153407] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 768.153767] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] self.force_reraise() [ 768.153767] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 768.153767] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] raise self.value [ 768.153767] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 768.153767] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] updated_port = self._update_port( [ 768.153767] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 768.153767] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] _ensure_no_port_binding_failure(port) [ 768.153767] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 768.153767] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] raise exception.PortBindingFailed(port_id=port['id']) [ 768.153767] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] nova.exception.PortBindingFailed: Binding failed for port 260fff8f-aefb-4420-a11a-81a330ebc23c, please check neutron logs for more information. [ 768.153767] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] [ 768.153767] env[64020]: INFO nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Terminating instance [ 768.154943] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Acquiring lock "refresh_cache-47516f85-60d8-4931-a205-b2ec1f7aec83" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 768.155097] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Acquired lock "refresh_cache-47516f85-60d8-4931-a205-b2ec1f7aec83" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 768.155258] env[64020]: DEBUG nova.network.neutron [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 768.155665] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg bb9944d82fcc4a8f90ae595ed4bbcea5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 768.162758] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb9944d82fcc4a8f90ae595ed4bbcea5 [ 768.261759] env[64020]: DEBUG nova.compute.manager [req-6343d57b-a2eb-48d1-87dd-65e6f06a5e5c req-dca32ffd-09f4-4b51-a8a1-f7efd6869466 service nova] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Received event network-changed-260fff8f-aefb-4420-a11a-81a330ebc23c {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 768.261960] env[64020]: DEBUG nova.compute.manager [req-6343d57b-a2eb-48d1-87dd-65e6f06a5e5c req-dca32ffd-09f4-4b51-a8a1-f7efd6869466 service nova] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Refreshing instance network info cache due to event network-changed-260fff8f-aefb-4420-a11a-81a330ebc23c. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 768.262124] env[64020]: DEBUG oslo_concurrency.lockutils [req-6343d57b-a2eb-48d1-87dd-65e6f06a5e5c req-dca32ffd-09f4-4b51-a8a1-f7efd6869466 service nova] Acquiring lock "refresh_cache-47516f85-60d8-4931-a205-b2ec1f7aec83" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 768.555783] env[64020]: DEBUG nova.network.neutron [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 768.610335] env[64020]: DEBUG nova.network.neutron [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 768.610335] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 66647f75ceaa4b869e5bc9824fd99a74 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 768.618669] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66647f75ceaa4b869e5bc9824fd99a74 [ 768.674914] env[64020]: DEBUG nova.network.neutron [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 768.753214] env[64020]: DEBUG nova.network.neutron [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 768.753850] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg fe97298ec35d4e64ab890fbb607b8f24 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 768.763464] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe97298ec35d4e64ab890fbb607b8f24 [ 768.849911] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e7b5e45-3c0c-4b7c-8cb0-384661ecc22b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.857598] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a9fa475-6449-4ecf-864d-6d1a025fb4ef {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.888724] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-353ab010-7699-4787-aca4-a57d6f53686a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.895980] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe0d4c52-0d9f-452d-80fd-f52bd2094179 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.911990] env[64020]: DEBUG nova.compute.provider_tree [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 768.912706] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg af227a314f7447cf8b5de5dc9dce1663 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 768.920065] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af227a314f7447cf8b5de5dc9dce1663 [ 769.113940] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Releasing lock "refresh_cache-866f8ba4-faad-4883-80ca-4c4dedb2b50c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 769.114198] env[64020]: DEBUG nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 769.114368] env[64020]: DEBUG nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 769.114529] env[64020]: DEBUG nova.network.neutron [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 769.130365] env[64020]: DEBUG nova.network.neutron [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 769.130928] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg c2eeb3c5718c4b238da09a86e0ec2569 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 769.138225] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2eeb3c5718c4b238da09a86e0ec2569 [ 769.256371] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Releasing lock "refresh_cache-47516f85-60d8-4931-a205-b2ec1f7aec83" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 769.256797] env[64020]: DEBUG nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 769.256982] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 769.257279] env[64020]: DEBUG oslo_concurrency.lockutils [req-6343d57b-a2eb-48d1-87dd-65e6f06a5e5c req-dca32ffd-09f4-4b51-a8a1-f7efd6869466 service nova] Acquired lock "refresh_cache-47516f85-60d8-4931-a205-b2ec1f7aec83" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 769.257441] env[64020]: DEBUG nova.network.neutron [req-6343d57b-a2eb-48d1-87dd-65e6f06a5e5c req-dca32ffd-09f4-4b51-a8a1-f7efd6869466 service nova] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Refreshing network info cache for port 260fff8f-aefb-4420-a11a-81a330ebc23c {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 769.257860] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-6343d57b-a2eb-48d1-87dd-65e6f06a5e5c req-dca32ffd-09f4-4b51-a8a1-f7efd6869466 service nova] Expecting reply to msg 16e0b7e18a264cd3ba7c10bd50860418 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 769.258667] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ab4dca02-8ea2-4932-ae3e-664550ceb52e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.267765] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-514241a1-9d74-4901-8398-bf87f6f1b14f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.277845] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16e0b7e18a264cd3ba7c10bd50860418 [ 769.289949] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 47516f85-60d8-4931-a205-b2ec1f7aec83 could not be found. [ 769.290122] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 769.290300] env[64020]: INFO nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Took 0.03 seconds to destroy the instance on the hypervisor. [ 769.290533] env[64020]: DEBUG oslo.service.loopingcall [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 769.290757] env[64020]: DEBUG nova.compute.manager [-] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 769.290852] env[64020]: DEBUG nova.network.neutron [-] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 769.305898] env[64020]: DEBUG nova.network.neutron [-] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 769.306492] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 29f5fe7b50204e5980ac51a9a3711e5e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 769.313396] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 29f5fe7b50204e5980ac51a9a3711e5e [ 769.415512] env[64020]: DEBUG nova.scheduler.client.report [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 769.417911] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg d36dc8a6738a4565926e17e7dc8b54f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 769.449135] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d36dc8a6738a4565926e17e7dc8b54f5 [ 769.633237] env[64020]: DEBUG nova.network.neutron [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 769.633498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 65f8d0752cd94634a152569d1e1b12bd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 769.641818] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65f8d0752cd94634a152569d1e1b12bd [ 769.778521] env[64020]: DEBUG nova.network.neutron [req-6343d57b-a2eb-48d1-87dd-65e6f06a5e5c req-dca32ffd-09f4-4b51-a8a1-f7efd6869466 service nova] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 769.810142] env[64020]: DEBUG nova.network.neutron [-] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 769.810142] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 27751d4198864367a8c7d640f0b17b7d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 769.817842] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27751d4198864367a8c7d640f0b17b7d [ 769.849533] env[64020]: DEBUG nova.network.neutron [req-6343d57b-a2eb-48d1-87dd-65e6f06a5e5c req-dca32ffd-09f4-4b51-a8a1-f7efd6869466 service nova] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 769.849533] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-6343d57b-a2eb-48d1-87dd-65e6f06a5e5c req-dca32ffd-09f4-4b51-a8a1-f7efd6869466 service nova] Expecting reply to msg ed4a8aed66a14001b3a637057afeabbe in queue reply_57893177120949e6a93cb88e15cd42b4 [ 769.857073] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed4a8aed66a14001b3a637057afeabbe [ 769.921137] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.889s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.921831] env[64020]: ERROR nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1f4c9a60-7cd0-4a45-be03-9f0bb35b2353, please check neutron logs for more information. [ 769.921831] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Traceback (most recent call last): [ 769.921831] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 769.921831] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] self.driver.spawn(context, instance, image_meta, [ 769.921831] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 769.921831] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 769.921831] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 769.921831] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] vm_ref = self.build_virtual_machine(instance, [ 769.921831] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 769.921831] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] vif_infos = vmwarevif.get_vif_info(self._session, [ 769.921831] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] for vif in network_info: [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] return self._sync_wrapper(fn, *args, **kwargs) [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] self.wait() [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] self[:] = self._gt.wait() [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] return self._exit_event.wait() [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] current.throw(*self._exc) [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 769.922196] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] result = function(*args, **kwargs) [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] return func(*args, **kwargs) [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] raise e [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] nwinfo = self.network_api.allocate_for_instance( [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] created_port_ids = self._update_ports_for_instance( [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] with excutils.save_and_reraise_exception(): [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] self.force_reraise() [ 769.922581] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 769.922973] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] raise self.value [ 769.922973] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 769.922973] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] updated_port = self._update_port( [ 769.922973] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 769.922973] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] _ensure_no_port_binding_failure(port) [ 769.922973] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 769.922973] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] raise exception.PortBindingFailed(port_id=port['id']) [ 769.922973] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] nova.exception.PortBindingFailed: Binding failed for port 1f4c9a60-7cd0-4a45-be03-9f0bb35b2353, please check neutron logs for more information. [ 769.922973] env[64020]: ERROR nova.compute.manager [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] [ 769.922973] env[64020]: DEBUG nova.compute.utils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Binding failed for port 1f4c9a60-7cd0-4a45-be03-9f0bb35b2353, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 769.924268] env[64020]: DEBUG nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Build of instance 371d5fb1-f1b6-41de-a4e6-88e8137f4be9 was re-scheduled: Binding failed for port 1f4c9a60-7cd0-4a45-be03-9f0bb35b2353, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 769.924667] env[64020]: DEBUG nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 769.924896] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Acquiring lock "refresh_cache-371d5fb1-f1b6-41de-a4e6-88e8137f4be9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 769.925041] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Acquired lock "refresh_cache-371d5fb1-f1b6-41de-a4e6-88e8137f4be9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 769.925196] env[64020]: DEBUG nova.network.neutron [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 769.925606] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg 14522dfa2c6c4dcab33c7815c7ec5f80 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 769.926747] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.442s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.928751] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg c37d03cefd114061b664f68fe75024bb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 769.932204] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14522dfa2c6c4dcab33c7815c7ec5f80 [ 769.957655] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c37d03cefd114061b664f68fe75024bb [ 770.135451] env[64020]: INFO nova.compute.manager [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] [instance: 866f8ba4-faad-4883-80ca-4c4dedb2b50c] Took 1.02 seconds to deallocate network for instance. [ 770.137175] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 1cb406099e7b4a8fa7b4fe5a8a5b80ac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 770.174921] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1cb406099e7b4a8fa7b4fe5a8a5b80ac [ 770.311691] env[64020]: INFO nova.compute.manager [-] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Took 1.02 seconds to deallocate network for instance. [ 770.314406] env[64020]: DEBUG nova.compute.claims [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 770.314582] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 770.350861] env[64020]: DEBUG oslo_concurrency.lockutils [req-6343d57b-a2eb-48d1-87dd-65e6f06a5e5c req-dca32ffd-09f4-4b51-a8a1-f7efd6869466 service nova] Releasing lock "refresh_cache-47516f85-60d8-4931-a205-b2ec1f7aec83" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 770.351186] env[64020]: DEBUG nova.compute.manager [req-6343d57b-a2eb-48d1-87dd-65e6f06a5e5c req-dca32ffd-09f4-4b51-a8a1-f7efd6869466 service nova] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Received event network-vif-deleted-260fff8f-aefb-4420-a11a-81a330ebc23c {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 770.443134] env[64020]: DEBUG nova.network.neutron [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 770.516126] env[64020]: DEBUG nova.network.neutron [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 770.516666] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg c9d5237d487c4f0cbb8a25a6a491bb60 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 770.525088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c9d5237d487c4f0cbb8a25a6a491bb60 [ 770.642042] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 45dbb98de17b489bb77dbf1bae7a1a0e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 770.674161] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 45dbb98de17b489bb77dbf1bae7a1a0e [ 770.726400] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64ddd461-8b1b-4ebe-b60a-ab22a9c7da73 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.733854] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-112100df-7fe1-414a-bc6f-dbc7e85bcf0a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.764313] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d075d809-e31c-4164-a976-9f51005a1e8f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.771506] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90c891ce-d908-4589-884a-f008053ab8b2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.784311] env[64020]: DEBUG nova.compute.provider_tree [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 770.784840] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg abf743bb07e94c50919537e314d05736 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 770.791556] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abf743bb07e94c50919537e314d05736 [ 771.020342] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Releasing lock "refresh_cache-371d5fb1-f1b6-41de-a4e6-88e8137f4be9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 771.020592] env[64020]: DEBUG nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 771.020779] env[64020]: DEBUG nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 771.020942] env[64020]: DEBUG nova.network.neutron [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 771.035993] env[64020]: DEBUG nova.network.neutron [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 771.036561] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg cc7238f9a9e3482195992c11e543f0a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 771.043363] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cc7238f9a9e3482195992c11e543f0a2 [ 771.167983] env[64020]: INFO nova.scheduler.client.report [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Deleted allocations for instance 866f8ba4-faad-4883-80ca-4c4dedb2b50c [ 771.174743] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Expecting reply to msg 3c9584098739404fbc9324ba0cdda310 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 771.190801] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c9584098739404fbc9324ba0cdda310 [ 771.286863] env[64020]: DEBUG nova.scheduler.client.report [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 771.289422] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 05faabe3f138430abc0d5f8e089a647c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 771.300137] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05faabe3f138430abc0d5f8e089a647c [ 771.539020] env[64020]: DEBUG nova.network.neutron [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 771.539589] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg d131d2794d164117b8a1d745f9f82f5e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 771.550556] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d131d2794d164117b8a1d745f9f82f5e [ 771.676274] env[64020]: DEBUG oslo_concurrency.lockutils [None req-01c64a29-5bbf-4988-8d94-a9da81f2972f tempest-MigrationsAdminTest-1352797037 tempest-MigrationsAdminTest-1352797037-project-member] Lock "866f8ba4-faad-4883-80ca-4c4dedb2b50c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 146.243s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 771.677051] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg a88a1783e5a249bfaa9e225882dc9611 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 771.687769] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a88a1783e5a249bfaa9e225882dc9611 [ 771.792326] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.865s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 771.793114] env[64020]: ERROR nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f27b0ef4-53c6-4a55-be67-1ced3268cfbc, please check neutron logs for more information. [ 771.793114] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Traceback (most recent call last): [ 771.793114] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 771.793114] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] self.driver.spawn(context, instance, image_meta, [ 771.793114] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 771.793114] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 771.793114] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 771.793114] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] vm_ref = self.build_virtual_machine(instance, [ 771.793114] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 771.793114] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] vif_infos = vmwarevif.get_vif_info(self._session, [ 771.793114] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] for vif in network_info: [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] return self._sync_wrapper(fn, *args, **kwargs) [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] self.wait() [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] self[:] = self._gt.wait() [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] return self._exit_event.wait() [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] current.throw(*self._exc) [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 771.793616] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] result = function(*args, **kwargs) [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] return func(*args, **kwargs) [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] raise e [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] nwinfo = self.network_api.allocate_for_instance( [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] created_port_ids = self._update_ports_for_instance( [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] with excutils.save_and_reraise_exception(): [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] self.force_reraise() [ 771.793953] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 771.794284] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] raise self.value [ 771.794284] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 771.794284] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] updated_port = self._update_port( [ 771.794284] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 771.794284] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] _ensure_no_port_binding_failure(port) [ 771.794284] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 771.794284] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] raise exception.PortBindingFailed(port_id=port['id']) [ 771.794284] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] nova.exception.PortBindingFailed: Binding failed for port f27b0ef4-53c6-4a55-be67-1ced3268cfbc, please check neutron logs for more information. [ 771.794284] env[64020]: ERROR nova.compute.manager [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] [ 771.794523] env[64020]: DEBUG nova.compute.utils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Binding failed for port f27b0ef4-53c6-4a55-be67-1ced3268cfbc, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 771.795612] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.657s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 771.797463] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg b06b3df9bf5e447c9a4465fd1e41de93 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 771.798675] env[64020]: DEBUG nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Build of instance a26baa75-d36d-4c05-bde8-d39fe814191d was re-scheduled: Binding failed for port f27b0ef4-53c6-4a55-be67-1ced3268cfbc, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 771.799092] env[64020]: DEBUG nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 771.799311] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "refresh_cache-a26baa75-d36d-4c05-bde8-d39fe814191d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 771.799455] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquired lock "refresh_cache-a26baa75-d36d-4c05-bde8-d39fe814191d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 771.799609] env[64020]: DEBUG nova.network.neutron [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 771.799966] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 7d27d2d40ad9435fbc2804e0b064f462 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 771.805904] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d27d2d40ad9435fbc2804e0b064f462 [ 771.828109] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b06b3df9bf5e447c9a4465fd1e41de93 [ 772.042518] env[64020]: INFO nova.compute.manager [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] [instance: 371d5fb1-f1b6-41de-a4e6-88e8137f4be9] Took 1.02 seconds to deallocate network for instance. [ 772.044269] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg 680c752755c94682a6ff180f6b234097 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 772.078163] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 680c752755c94682a6ff180f6b234097 [ 772.179524] env[64020]: DEBUG nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 772.182009] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 86aea00aacc24e9c8da1bd5407e98e57 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 772.218527] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86aea00aacc24e9c8da1bd5407e98e57 [ 772.318693] env[64020]: DEBUG nova.network.neutron [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 772.413065] env[64020]: DEBUG nova.network.neutron [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 772.413931] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 083a65cf7df74c078411c1fe64b7d5ac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 772.423458] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 083a65cf7df74c078411c1fe64b7d5ac [ 772.556353] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg 4c289d462c5947a79d528eea24b64daf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 772.594456] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c289d462c5947a79d528eea24b64daf [ 772.696507] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 772.714483] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f90fdf97-a000-4ddf-9e18-6243053dd0c2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.722290] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2d15f12-68ca-4b45-affa-68712bbd5087 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.753672] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae92c2fa-0d2e-48a9-a3a0-f21362645bae {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.761236] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c62efbc9-1379-4a8a-b03a-6411320f0d4d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.777739] env[64020]: DEBUG nova.compute.provider_tree [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 772.778241] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg df96ca7eef6041d4b7af0cbda592a392 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 772.791164] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg df96ca7eef6041d4b7af0cbda592a392 [ 772.917065] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Releasing lock "refresh_cache-a26baa75-d36d-4c05-bde8-d39fe814191d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 772.917306] env[64020]: DEBUG nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 772.917481] env[64020]: DEBUG nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 772.917681] env[64020]: DEBUG nova.network.neutron [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 772.939063] env[64020]: DEBUG nova.network.neutron [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 772.939640] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 6cdbbd1616a94cf48952ea80e9dc8b3b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 772.948208] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6cdbbd1616a94cf48952ea80e9dc8b3b [ 773.082346] env[64020]: INFO nova.scheduler.client.report [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Deleted allocations for instance 371d5fb1-f1b6-41de-a4e6-88e8137f4be9 [ 773.098226] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Expecting reply to msg 6dd27cf0d2e441fc866940bea5e4e0b0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 773.121591] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6dd27cf0d2e441fc866940bea5e4e0b0 [ 773.287575] env[64020]: DEBUG nova.scheduler.client.report [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 773.290179] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 6869f76517304abe8e7bd6d6b116f01c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 773.301263] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6869f76517304abe8e7bd6d6b116f01c [ 773.441947] env[64020]: DEBUG nova.network.neutron [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 773.442512] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 18842572b8c74f3c8154952ad2539e3b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 773.452773] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18842572b8c74f3c8154952ad2539e3b [ 773.603653] env[64020]: DEBUG oslo_concurrency.lockutils [None req-2a37afd0-bdcf-4b4d-b41d-25428392deee tempest-FloatingIPsAssociationTestJSON-1504857872 tempest-FloatingIPsAssociationTestJSON-1504857872-project-member] Lock "371d5fb1-f1b6-41de-a4e6-88e8137f4be9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 145.822s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 773.604072] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg a31b6f1ce4e74713ab113bdc35c0e0f7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 773.617088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a31b6f1ce4e74713ab113bdc35c0e0f7 [ 773.794983] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.999s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 773.795644] env[64020]: ERROR nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 829381ab-a31a-4ad8-a28d-ace00c45d6c8, please check neutron logs for more information. [ 773.795644] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Traceback (most recent call last): [ 773.795644] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 773.795644] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] self.driver.spawn(context, instance, image_meta, [ 773.795644] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 773.795644] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] self._vmops.spawn(context, instance, image_meta, injected_files, [ 773.795644] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 773.795644] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] vm_ref = self.build_virtual_machine(instance, [ 773.795644] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 773.795644] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] vif_infos = vmwarevif.get_vif_info(self._session, [ 773.795644] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] for vif in network_info: [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] return self._sync_wrapper(fn, *args, **kwargs) [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] self.wait() [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] self[:] = self._gt.wait() [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] return self._exit_event.wait() [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] current.throw(*self._exc) [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 773.796243] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] result = function(*args, **kwargs) [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] return func(*args, **kwargs) [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] raise e [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] nwinfo = self.network_api.allocate_for_instance( [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] created_port_ids = self._update_ports_for_instance( [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] with excutils.save_and_reraise_exception(): [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] self.force_reraise() [ 773.796717] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 773.797207] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] raise self.value [ 773.797207] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 773.797207] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] updated_port = self._update_port( [ 773.797207] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 773.797207] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] _ensure_no_port_binding_failure(port) [ 773.797207] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 773.797207] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] raise exception.PortBindingFailed(port_id=port['id']) [ 773.797207] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] nova.exception.PortBindingFailed: Binding failed for port 829381ab-a31a-4ad8-a28d-ace00c45d6c8, please check neutron logs for more information. [ 773.797207] env[64020]: ERROR nova.compute.manager [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] [ 773.797207] env[64020]: DEBUG nova.compute.utils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Binding failed for port 829381ab-a31a-4ad8-a28d-ace00c45d6c8, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 773.797623] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.664s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 773.799491] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg bd4021dfa0c942deba73a9183abd0987 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 773.800661] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Build of instance e01583c9-744c-47ea-b7cf-a46b046fc854 was re-scheduled: Binding failed for port 829381ab-a31a-4ad8-a28d-ace00c45d6c8, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 773.801017] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 773.801246] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "refresh_cache-e01583c9-744c-47ea-b7cf-a46b046fc854" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 773.801389] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquired lock "refresh_cache-e01583c9-744c-47ea-b7cf-a46b046fc854" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 773.801580] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 773.801979] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 65a29f468e8049b4a1d4fe61df4060e3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 773.813871] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65a29f468e8049b4a1d4fe61df4060e3 [ 773.838671] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bd4021dfa0c942deba73a9183abd0987 [ 773.946923] env[64020]: INFO nova.compute.manager [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: a26baa75-d36d-4c05-bde8-d39fe814191d] Took 1.03 seconds to deallocate network for instance. [ 773.948797] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 233ab2b74be24c1d8d01095dc368f1cd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 773.994173] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 233ab2b74be24c1d8d01095dc368f1cd [ 774.106697] env[64020]: DEBUG nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 774.108538] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 875e8fcbe0a547d4b8a3196894a15b6d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 774.155683] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 875e8fcbe0a547d4b8a3196894a15b6d [ 774.332340] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 774.444469] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 774.444994] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 70d4bae4b8b844bc8f39441bab891641 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 774.453198] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg eebc786dde9c40fa871438ee095320a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 774.497376] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eebc786dde9c40fa871438ee095320a2 [ 774.497527] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 70d4bae4b8b844bc8f39441bab891641 [ 774.627867] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 774.688837] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fbaabc90-f54b-41ad-b243-118f3af005e4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.699110] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5256a35-c0ee-41f4-95cc-1df2a879cdb7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.732841] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a3f6c94-c5e4-4011-890d-6a0d676baed2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.742188] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d58b884-7ced-4ab1-ab6a-9e046a86dce8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.755342] env[64020]: DEBUG nova.compute.provider_tree [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 774.755948] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg d56a3583a85b474c9f7ca54578409a1f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 774.769886] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d56a3583a85b474c9f7ca54578409a1f [ 774.947507] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Releasing lock "refresh_cache-e01583c9-744c-47ea-b7cf-a46b046fc854" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 774.947507] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 774.947607] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 774.947930] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 774.969391] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 774.970009] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 222b77af03b548b593f531f5de6734e0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 774.979634] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 222b77af03b548b593f531f5de6734e0 [ 774.994757] env[64020]: INFO nova.scheduler.client.report [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Deleted allocations for instance a26baa75-d36d-4c05-bde8-d39fe814191d [ 775.010187] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 68d82d8e71504e3d80aa3c46a80688c4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 775.026771] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 68d82d8e71504e3d80aa3c46a80688c4 [ 775.259236] env[64020]: DEBUG nova.scheduler.client.report [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 775.262497] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 40a649acd5d84f01ad4ecfafc2b624f9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 775.273096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40a649acd5d84f01ad4ecfafc2b624f9 [ 775.299353] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Acquiring lock "f4682b82-7475-412b-9319-bee1f05a9c63" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 775.299577] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Lock "f4682b82-7475-412b-9319-bee1f05a9c63" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 775.473103] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 775.473685] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg f8abcf1ed2104fec8e00deb566bcf446 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 775.482930] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f8abcf1ed2104fec8e00deb566bcf446 [ 775.511972] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d39cfa5c-0f9b-423b-92fd-619de0e34ba1 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "a26baa75-d36d-4c05-bde8-d39fe814191d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 146.194s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 775.512720] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg a92d295f9f6e4d04b03f53768fe636b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 775.521897] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a92d295f9f6e4d04b03f53768fe636b3 [ 775.764718] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.967s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 775.765284] env[64020]: ERROR nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8, please check neutron logs for more information. [ 775.765284] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] Traceback (most recent call last): [ 775.765284] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 775.765284] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] self.driver.spawn(context, instance, image_meta, [ 775.765284] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 775.765284] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] self._vmops.spawn(context, instance, image_meta, injected_files, [ 775.765284] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 775.765284] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] vm_ref = self.build_virtual_machine(instance, [ 775.765284] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 775.765284] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] vif_infos = vmwarevif.get_vif_info(self._session, [ 775.765284] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] for vif in network_info: [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] return self._sync_wrapper(fn, *args, **kwargs) [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] self.wait() [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] self[:] = self._gt.wait() [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] return self._exit_event.wait() [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] current.throw(*self._exc) [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 775.765743] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] result = function(*args, **kwargs) [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] return func(*args, **kwargs) [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] raise e [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] nwinfo = self.network_api.allocate_for_instance( [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] created_port_ids = self._update_ports_for_instance( [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] with excutils.save_and_reraise_exception(): [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] self.force_reraise() [ 775.766110] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 775.766561] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] raise self.value [ 775.766561] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 775.766561] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] updated_port = self._update_port( [ 775.766561] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 775.766561] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] _ensure_no_port_binding_failure(port) [ 775.766561] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 775.766561] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] raise exception.PortBindingFailed(port_id=port['id']) [ 775.766561] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] nova.exception.PortBindingFailed: Binding failed for port c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8, please check neutron logs for more information. [ 775.766561] env[64020]: ERROR nova.compute.manager [instance: 786a572c-adaa-4656-8b42-c510940dac59] [ 775.766561] env[64020]: DEBUG nova.compute.utils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Binding failed for port c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 775.767271] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.909s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 775.769084] env[64020]: INFO nova.compute.claims [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 775.770666] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg dde181af0121478da1859a006a4f1d12 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 775.772324] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Build of instance 786a572c-adaa-4656-8b42-c510940dac59 was re-scheduled: Binding failed for port c1da85ce-6886-4c05-8b92-ea4ea6f0f5c8, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 775.772791] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 775.773013] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "refresh_cache-786a572c-adaa-4656-8b42-c510940dac59" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 775.773158] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquired lock "refresh_cache-786a572c-adaa-4656-8b42-c510940dac59" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 775.773314] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 775.773664] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg f0fbce6cbc8a4728bf9aa73f588c0d1a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 775.780326] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f0fbce6cbc8a4728bf9aa73f588c0d1a [ 775.819927] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dde181af0121478da1859a006a4f1d12 [ 775.977793] env[64020]: INFO nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: e01583c9-744c-47ea-b7cf-a46b046fc854] Took 1.03 seconds to deallocate network for instance. [ 775.979540] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 3355dbe4d9d04989993229743a239635 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 776.018573] env[64020]: DEBUG nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 776.018573] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 93a1535aeba446179a2db4f552fdb7ef in queue reply_57893177120949e6a93cb88e15cd42b4 [ 776.258716] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3355dbe4d9d04989993229743a239635 [ 776.273837] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 4dc590084e2346749411743e4d11802f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 776.279558] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93a1535aeba446179a2db4f552fdb7ef [ 776.284836] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4dc590084e2346749411743e4d11802f [ 776.289143] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 776.385661] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 776.386179] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg ea9d9dded6e74b569e0205db1a879086 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 776.394210] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ea9d9dded6e74b569e0205db1a879086 [ 776.491277] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg e0d5c7ed25174621b34e85815cc7fd43 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 776.527486] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e0d5c7ed25174621b34e85815cc7fd43 [ 776.538538] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 776.890850] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Releasing lock "refresh_cache-786a572c-adaa-4656-8b42-c510940dac59" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 776.891080] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 776.891291] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 776.891463] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 776.913773] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 776.914357] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 9984df0692e04558a4bc55ae87881be9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 776.924620] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9984df0692e04558a4bc55ae87881be9 [ 777.011720] env[64020]: INFO nova.scheduler.client.report [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Deleted allocations for instance e01583c9-744c-47ea-b7cf-a46b046fc854 [ 777.020803] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 51304b6178dc4b209c0e2707da9b596c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 777.037373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51304b6178dc4b209c0e2707da9b596c [ 777.143445] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f74d22a-ca02-44c4-9877-94a842d22524 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 777.151546] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9380774-290d-4d59-b571-97839435551f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 777.180086] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09c7d043-2ac7-4b9b-88da-f41f7ca341df {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 777.186963] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71010776-82a4-4ea4-8b7d-afa63592c169 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 777.200345] env[64020]: DEBUG nova.compute.provider_tree [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 777.200859] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg bbcbc171316946ac966eacabafd17a94 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 777.208279] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bbcbc171316946ac966eacabafd17a94 [ 777.420755] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 777.421466] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 4bc5e8c1f46f45faad23d5eac03e1f95 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 777.429933] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4bc5e8c1f46f45faad23d5eac03e1f95 [ 777.527462] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "e01583c9-744c-47ea-b7cf-a46b046fc854" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 147.485s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 777.528102] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg bceb5a98504f40c9b9ab3bf19f783763 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 777.540484] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bceb5a98504f40c9b9ab3bf19f783763 [ 777.703383] env[64020]: DEBUG nova.scheduler.client.report [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 777.706072] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 404267ab27c0487ea1f467eeaf994847 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 777.718596] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 404267ab27c0487ea1f467eeaf994847 [ 777.923750] env[64020]: INFO nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 786a572c-adaa-4656-8b42-c510940dac59] Took 1.03 seconds to deallocate network for instance. [ 777.925565] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 87eeff7ab42240cdb34e6b94751c2e96 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 777.973836] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 87eeff7ab42240cdb34e6b94751c2e96 [ 778.033373] env[64020]: DEBUG nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 778.033373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 4fea4686818a42f5b6027ee0563dae48 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 778.074795] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4fea4686818a42f5b6027ee0563dae48 [ 778.209131] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.442s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 778.209694] env[64020]: DEBUG nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 778.211365] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 0da5e923dff0495d9b7c7b9a102277ae in queue reply_57893177120949e6a93cb88e15cd42b4 [ 778.212336] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.733s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 778.214001] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg ada8c3d4ceb24b90aa242544520c96c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 778.252373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0da5e923dff0495d9b7c7b9a102277ae [ 778.274197] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ada8c3d4ceb24b90aa242544520c96c1 [ 778.430552] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg ce744a94a16b46949e8fb8ca0c5183b0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 778.479379] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce744a94a16b46949e8fb8ca0c5183b0 [ 778.553132] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 778.717366] env[64020]: DEBUG nova.compute.utils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 778.717996] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg fef45fe184684f928341929917831d1d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 778.719030] env[64020]: DEBUG nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 778.719113] env[64020]: DEBUG nova.network.neutron [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 778.730752] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fef45fe184684f928341929917831d1d [ 778.795808] env[64020]: DEBUG nova.policy [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '92dbbb520a274d98baff4fb9a10c6a0d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'dcb43e39f8964efb9dde2ad826a224d4', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 778.956707] env[64020]: INFO nova.scheduler.client.report [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Deleted allocations for instance 786a572c-adaa-4656-8b42-c510940dac59 [ 778.967677] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg a6ecff0dc0e34a04b6c90de1d5f8c3cd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 779.007172] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6ecff0dc0e34a04b6c90de1d5f8c3cd [ 779.111014] env[64020]: DEBUG nova.network.neutron [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Successfully created port: 46ff2091-6fa7-4aac-bfa0-2fdb8087185d {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 779.125242] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58039ec0-eb0a-4162-8d76-c7bafd7b7c7a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.132810] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7cdc1440-deb0-4318-aea3-34e98c189427 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.164750] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8682a037-ee66-4e9a-b5c7-6d2b3851b7ce {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.172459] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5298c24-69a6-46fd-916f-3cbbeb85d73d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.194897] env[64020]: DEBUG nova.compute.provider_tree [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 779.195828] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg facaea2428ec41ca842d3123346a0dc7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 779.205160] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg facaea2428ec41ca842d3123346a0dc7 [ 779.222383] env[64020]: DEBUG nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 779.224207] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 58f8e4735b734ab4a7f3f010d3da8b18 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 779.266468] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 58f8e4735b734ab4a7f3f010d3da8b18 [ 779.470618] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "786a572c-adaa-4656-8b42-c510940dac59" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 149.379s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 779.470618] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 0f2e1d073ce545d3b9ceebef7f10f901 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 779.481170] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f2e1d073ce545d3b9ceebef7f10f901 [ 779.701961] env[64020]: DEBUG nova.scheduler.client.report [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 779.701961] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg cc117cb1b7c246c5962b28207b2248ff in queue reply_57893177120949e6a93cb88e15cd42b4 [ 779.731268] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg b04f14759ec241b0b92cafbf787ece6d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 779.734986] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cc117cb1b7c246c5962b28207b2248ff [ 779.775855] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b04f14759ec241b0b92cafbf787ece6d [ 779.827742] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "1235c526-3cfb-42b0-8e2d-64cfc59ec35e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 779.827982] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "1235c526-3cfb-42b0-8e2d-64cfc59ec35e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 779.960433] env[64020]: DEBUG nova.compute.manager [req-3ef8d39a-361b-4caf-8646-bfa378709848 req-1fc2ff76-e8ea-4fc5-a06d-40403206f0dd service nova] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Received event network-changed-46ff2091-6fa7-4aac-bfa0-2fdb8087185d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 779.961298] env[64020]: DEBUG nova.compute.manager [req-3ef8d39a-361b-4caf-8646-bfa378709848 req-1fc2ff76-e8ea-4fc5-a06d-40403206f0dd service nova] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Refreshing instance network info cache due to event network-changed-46ff2091-6fa7-4aac-bfa0-2fdb8087185d. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 779.961298] env[64020]: DEBUG oslo_concurrency.lockutils [req-3ef8d39a-361b-4caf-8646-bfa378709848 req-1fc2ff76-e8ea-4fc5-a06d-40403206f0dd service nova] Acquiring lock "refresh_cache-2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 779.961298] env[64020]: DEBUG oslo_concurrency.lockutils [req-3ef8d39a-361b-4caf-8646-bfa378709848 req-1fc2ff76-e8ea-4fc5-a06d-40403206f0dd service nova] Acquired lock "refresh_cache-2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 779.961298] env[64020]: DEBUG nova.network.neutron [req-3ef8d39a-361b-4caf-8646-bfa378709848 req-1fc2ff76-e8ea-4fc5-a06d-40403206f0dd service nova] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Refreshing network info cache for port 46ff2091-6fa7-4aac-bfa0-2fdb8087185d {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 779.961529] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3ef8d39a-361b-4caf-8646-bfa378709848 req-1fc2ff76-e8ea-4fc5-a06d-40403206f0dd service nova] Expecting reply to msg 753f4e29384749ffa60ee3394e563e50 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 779.972856] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 753f4e29384749ffa60ee3394e563e50 [ 779.973322] env[64020]: DEBUG nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 779.975187] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 6888fc23d20646eea07b546b0088bdda in queue reply_57893177120949e6a93cb88e15cd42b4 [ 780.016780] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6888fc23d20646eea07b546b0088bdda [ 780.079403] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Acquiring lock "8c4773b7-a8de-4de3-a91f-2252cc34d11b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 780.079614] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Lock "8c4773b7-a8de-4de3-a91f-2252cc34d11b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 780.182424] env[64020]: ERROR nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 46ff2091-6fa7-4aac-bfa0-2fdb8087185d, please check neutron logs for more information. [ 780.182424] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 780.182424] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 780.182424] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 780.182424] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 780.182424] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 780.182424] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 780.182424] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 780.182424] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 780.182424] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 780.182424] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 780.182424] env[64020]: ERROR nova.compute.manager raise self.value [ 780.182424] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 780.182424] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 780.182424] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 780.182424] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 780.183017] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 780.183017] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 780.183017] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 46ff2091-6fa7-4aac-bfa0-2fdb8087185d, please check neutron logs for more information. [ 780.183017] env[64020]: ERROR nova.compute.manager [ 780.183017] env[64020]: Traceback (most recent call last): [ 780.183017] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 780.183017] env[64020]: listener.cb(fileno) [ 780.183017] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 780.183017] env[64020]: result = function(*args, **kwargs) [ 780.183017] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 780.183017] env[64020]: return func(*args, **kwargs) [ 780.183017] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 780.183017] env[64020]: raise e [ 780.183017] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 780.183017] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 780.183017] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 780.183017] env[64020]: created_port_ids = self._update_ports_for_instance( [ 780.183017] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 780.183017] env[64020]: with excutils.save_and_reraise_exception(): [ 780.183017] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 780.183017] env[64020]: self.force_reraise() [ 780.183017] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 780.183017] env[64020]: raise self.value [ 780.183017] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 780.183017] env[64020]: updated_port = self._update_port( [ 780.183017] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 780.183017] env[64020]: _ensure_no_port_binding_failure(port) [ 780.183017] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 780.183017] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 780.183941] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 46ff2091-6fa7-4aac-bfa0-2fdb8087185d, please check neutron logs for more information. [ 780.183941] env[64020]: Removing descriptor: 18 [ 780.204667] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.992s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 780.205217] env[64020]: ERROR nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 25741d42-2714-4498-9df6-2f8af144e18b, please check neutron logs for more information. [ 780.205217] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Traceback (most recent call last): [ 780.205217] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 780.205217] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] self.driver.spawn(context, instance, image_meta, [ 780.205217] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 780.205217] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] self._vmops.spawn(context, instance, image_meta, injected_files, [ 780.205217] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 780.205217] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] vm_ref = self.build_virtual_machine(instance, [ 780.205217] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 780.205217] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] vif_infos = vmwarevif.get_vif_info(self._session, [ 780.205217] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] for vif in network_info: [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] return self._sync_wrapper(fn, *args, **kwargs) [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] self.wait() [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] self[:] = self._gt.wait() [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] return self._exit_event.wait() [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] current.throw(*self._exc) [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 780.205616] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] result = function(*args, **kwargs) [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] return func(*args, **kwargs) [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] raise e [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] nwinfo = self.network_api.allocate_for_instance( [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] created_port_ids = self._update_ports_for_instance( [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] with excutils.save_and_reraise_exception(): [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] self.force_reraise() [ 780.206032] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 780.206477] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] raise self.value [ 780.206477] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 780.206477] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] updated_port = self._update_port( [ 780.206477] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 780.206477] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] _ensure_no_port_binding_failure(port) [ 780.206477] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 780.206477] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] raise exception.PortBindingFailed(port_id=port['id']) [ 780.206477] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] nova.exception.PortBindingFailed: Binding failed for port 25741d42-2714-4498-9df6-2f8af144e18b, please check neutron logs for more information. [ 780.206477] env[64020]: ERROR nova.compute.manager [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] [ 780.206477] env[64020]: DEBUG nova.compute.utils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Binding failed for port 25741d42-2714-4498-9df6-2f8af144e18b, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 780.207144] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.828s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 780.208945] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 7bcbdadf84464f2bbb95042bc0e97b94 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 780.210068] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Build of instance 8e8e1b50-fcae-41c6-8f5a-241692140cda was re-scheduled: Binding failed for port 25741d42-2714-4498-9df6-2f8af144e18b, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 780.210498] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 780.210707] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquiring lock "refresh_cache-8e8e1b50-fcae-41c6-8f5a-241692140cda" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 780.210846] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Acquired lock "refresh_cache-8e8e1b50-fcae-41c6-8f5a-241692140cda" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 780.210993] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 780.211373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg abb0996f6fe0473e951de40590709183 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 780.218648] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abb0996f6fe0473e951de40590709183 [ 780.235935] env[64020]: DEBUG nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 780.253862] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7bcbdadf84464f2bbb95042bc0e97b94 [ 780.265946] env[64020]: DEBUG nova.virt.hardware [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 780.266221] env[64020]: DEBUG nova.virt.hardware [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 780.266381] env[64020]: DEBUG nova.virt.hardware [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 780.266557] env[64020]: DEBUG nova.virt.hardware [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 780.266698] env[64020]: DEBUG nova.virt.hardware [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 780.266840] env[64020]: DEBUG nova.virt.hardware [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 780.267043] env[64020]: DEBUG nova.virt.hardware [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 780.267192] env[64020]: DEBUG nova.virt.hardware [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 780.267349] env[64020]: DEBUG nova.virt.hardware [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 780.267523] env[64020]: DEBUG nova.virt.hardware [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 780.267707] env[64020]: DEBUG nova.virt.hardware [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 780.268792] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a041ac3a-9c56-4937-a453-cafde604cb24 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 780.276799] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28c9b999-0b84-4f80-9dab-0625244ad362 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 780.292124] env[64020]: ERROR nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 46ff2091-6fa7-4aac-bfa0-2fdb8087185d, please check neutron logs for more information. [ 780.292124] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Traceback (most recent call last): [ 780.292124] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 780.292124] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] yield resources [ 780.292124] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 780.292124] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] self.driver.spawn(context, instance, image_meta, [ 780.292124] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 780.292124] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] self._vmops.spawn(context, instance, image_meta, injected_files, [ 780.292124] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 780.292124] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] vm_ref = self.build_virtual_machine(instance, [ 780.292124] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] vif_infos = vmwarevif.get_vif_info(self._session, [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] for vif in network_info: [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] return self._sync_wrapper(fn, *args, **kwargs) [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] self.wait() [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] self[:] = self._gt.wait() [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] return self._exit_event.wait() [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 780.292612] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] current.throw(*self._exc) [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] result = function(*args, **kwargs) [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] return func(*args, **kwargs) [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] raise e [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] nwinfo = self.network_api.allocate_for_instance( [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] created_port_ids = self._update_ports_for_instance( [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] with excutils.save_and_reraise_exception(): [ 780.292992] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 780.293369] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] self.force_reraise() [ 780.293369] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 780.293369] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] raise self.value [ 780.293369] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 780.293369] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] updated_port = self._update_port( [ 780.293369] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 780.293369] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] _ensure_no_port_binding_failure(port) [ 780.293369] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 780.293369] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] raise exception.PortBindingFailed(port_id=port['id']) [ 780.293369] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] nova.exception.PortBindingFailed: Binding failed for port 46ff2091-6fa7-4aac-bfa0-2fdb8087185d, please check neutron logs for more information. [ 780.293369] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] [ 780.293369] env[64020]: INFO nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Terminating instance [ 780.294442] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquiring lock "refresh_cache-2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 780.481144] env[64020]: DEBUG nova.network.neutron [req-3ef8d39a-361b-4caf-8646-bfa378709848 req-1fc2ff76-e8ea-4fc5-a06d-40403206f0dd service nova] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 780.503805] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 780.556827] env[64020]: DEBUG nova.network.neutron [req-3ef8d39a-361b-4caf-8646-bfa378709848 req-1fc2ff76-e8ea-4fc5-a06d-40403206f0dd service nova] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 780.557354] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3ef8d39a-361b-4caf-8646-bfa378709848 req-1fc2ff76-e8ea-4fc5-a06d-40403206f0dd service nova] Expecting reply to msg db44aae456c34923b263a33c0c299684 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 780.570144] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db44aae456c34923b263a33c0c299684 [ 780.736272] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 780.811038] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 780.811632] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 43614f3f66fe49e795223dc00e74bea6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 780.820081] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43614f3f66fe49e795223dc00e74bea6 [ 781.061205] env[64020]: DEBUG oslo_concurrency.lockutils [req-3ef8d39a-361b-4caf-8646-bfa378709848 req-1fc2ff76-e8ea-4fc5-a06d-40403206f0dd service nova] Releasing lock "refresh_cache-2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 781.063956] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquired lock "refresh_cache-2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 781.064164] env[64020]: DEBUG nova.network.neutron [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 781.064597] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 3feefd09e7994f10a02a7e473e56210f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 781.072717] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3feefd09e7994f10a02a7e473e56210f [ 781.101725] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73c8b8b7-dac1-4153-a45f-7d6c0c829fa1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 781.109836] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4782b64-718d-42e8-9038-54aa46962817 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 781.140174] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8bfaeebd-0e20-4ee6-a908-45fa4cd2b442 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 781.147536] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69fea846-98d8-458d-8fa2-4a5f79659d0e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 781.162344] env[64020]: DEBUG nova.compute.provider_tree [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 781.162832] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg c90ddd4f62bc400697b96d8740ac8f39 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 781.170368] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c90ddd4f62bc400697b96d8740ac8f39 [ 781.314306] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Releasing lock "refresh_cache-8e8e1b50-fcae-41c6-8f5a-241692140cda" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 781.314668] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 781.314883] env[64020]: DEBUG nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 781.315048] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 781.330797] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 781.331468] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg d8fa227915d34ae392532b5ea8121c49 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 781.338389] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d8fa227915d34ae392532b5ea8121c49 [ 781.584490] env[64020]: DEBUG nova.network.neutron [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 781.634159] env[64020]: DEBUG nova.network.neutron [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 781.634680] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg a9f3e9e5d2bb4adcb305f5853aeff977 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 781.643060] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a9f3e9e5d2bb4adcb305f5853aeff977 [ 781.665151] env[64020]: DEBUG nova.scheduler.client.report [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 781.667532] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 5fb0627e4df7452b9a42694f0fde6407 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 781.680636] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5fb0627e4df7452b9a42694f0fde6407 [ 781.834245] env[64020]: DEBUG nova.network.neutron [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 781.834723] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg e7a1e1d8edca4b3d9794e499d5d90b6b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 781.843265] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e7a1e1d8edca4b3d9794e499d5d90b6b [ 782.005575] env[64020]: DEBUG nova.compute.manager [req-82c4d6dd-d911-40dc-a8b8-96b95d79635a req-ebf7875b-33e5-491f-81c6-3bb43eafbf15 service nova] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Received event network-vif-deleted-46ff2091-6fa7-4aac-bfa0-2fdb8087185d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 782.137400] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Releasing lock "refresh_cache-2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 782.137848] env[64020]: DEBUG nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 782.138033] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 782.138391] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c59caabd-7d15-463d-8459-483ef0bf32ec {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 782.147318] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ecf80f12-9a63-4fa0-9e5b-77ea322cf34e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 782.167775] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa could not be found. [ 782.167977] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 782.168163] env[64020]: INFO nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Took 0.03 seconds to destroy the instance on the hypervisor. [ 782.168446] env[64020]: DEBUG oslo.service.loopingcall [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 782.168646] env[64020]: DEBUG nova.compute.manager [-] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 782.168739] env[64020]: DEBUG nova.network.neutron [-] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 782.170672] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.964s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.171244] env[64020]: ERROR nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1558d2ac-f57e-47e6-81d9-e75b57f46aee, please check neutron logs for more information. [ 782.171244] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Traceback (most recent call last): [ 782.171244] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 782.171244] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] self.driver.spawn(context, instance, image_meta, [ 782.171244] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 782.171244] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] self._vmops.spawn(context, instance, image_meta, injected_files, [ 782.171244] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 782.171244] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] vm_ref = self.build_virtual_machine(instance, [ 782.171244] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 782.171244] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] vif_infos = vmwarevif.get_vif_info(self._session, [ 782.171244] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] for vif in network_info: [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] return self._sync_wrapper(fn, *args, **kwargs) [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] self.wait() [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] self[:] = self._gt.wait() [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] return self._exit_event.wait() [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] result = hub.switch() [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 782.171614] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] return self.greenlet.switch() [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] result = function(*args, **kwargs) [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] return func(*args, **kwargs) [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] raise e [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] nwinfo = self.network_api.allocate_for_instance( [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] created_port_ids = self._update_ports_for_instance( [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] with excutils.save_and_reraise_exception(): [ 782.172046] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 782.172646] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] self.force_reraise() [ 782.172646] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 782.172646] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] raise self.value [ 782.172646] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 782.172646] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] updated_port = self._update_port( [ 782.172646] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 782.172646] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] _ensure_no_port_binding_failure(port) [ 782.172646] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 782.172646] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] raise exception.PortBindingFailed(port_id=port['id']) [ 782.172646] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] nova.exception.PortBindingFailed: Binding failed for port 1558d2ac-f57e-47e6-81d9-e75b57f46aee, please check neutron logs for more information. [ 782.172646] env[64020]: ERROR nova.compute.manager [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] [ 782.173229] env[64020]: DEBUG nova.compute.utils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Binding failed for port 1558d2ac-f57e-47e6-81d9-e75b57f46aee, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 782.173229] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.298s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.174703] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 4afc2baa3ec348b9bf893dc69aadff1f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 782.176733] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Build of instance d808804b-a2d0-4225-beb5-7d91e0cff3ac was re-scheduled: Binding failed for port 1558d2ac-f57e-47e6-81d9-e75b57f46aee, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 782.177144] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 782.177361] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "refresh_cache-d808804b-a2d0-4225-beb5-7d91e0cff3ac" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 782.177505] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquired lock "refresh_cache-d808804b-a2d0-4225-beb5-7d91e0cff3ac" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 782.177646] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 782.177997] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 52e83cfb3f0c471283b350236d544b6f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 782.184123] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52e83cfb3f0c471283b350236d544b6f [ 782.188607] env[64020]: DEBUG nova.network.neutron [-] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 782.189019] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3ab651881e7f46c89c32a6fde8a462b7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 782.203608] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3ab651881e7f46c89c32a6fde8a462b7 [ 782.210210] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4afc2baa3ec348b9bf893dc69aadff1f [ 782.337387] env[64020]: INFO nova.compute.manager [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] [instance: 8e8e1b50-fcae-41c6-8f5a-241692140cda] Took 1.02 seconds to deallocate network for instance. [ 782.339076] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg fdcc360b087c423382a89fd3ab99e383 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 782.374147] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fdcc360b087c423382a89fd3ab99e383 [ 782.690405] env[64020]: DEBUG nova.network.neutron [-] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 782.690824] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8b46fa87dba04aeaa142be9962f9b32b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 782.696808] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 782.701858] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b46fa87dba04aeaa142be9962f9b32b [ 782.776209] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 782.776711] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 366e00111938456bb8bc6ab950dea352 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 782.786709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 366e00111938456bb8bc6ab950dea352 [ 782.844039] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 589492d54b2c4f5d86d08f28959ce61e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 782.874465] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 589492d54b2c4f5d86d08f28959ce61e [ 782.992675] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-331222b2-4b13-441b-8acc-3bc0a940c65b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.000509] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b623bf2e-b6e0-44bc-a155-d57842d193f0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.031308] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f1906e0-0476-4f51-83c6-dd29b0ca0ba1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.038740] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6574afa7-5096-4dcf-98cf-f30245d80943 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.052184] env[64020]: DEBUG nova.compute.provider_tree [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 783.052904] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg ae1c25e80c96489aa7f869b0687bb9b2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 783.059742] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae1c25e80c96489aa7f869b0687bb9b2 [ 783.193091] env[64020]: INFO nova.compute.manager [-] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Took 1.02 seconds to deallocate network for instance. [ 783.196326] env[64020]: DEBUG nova.compute.claims [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 783.196326] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 783.279155] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Releasing lock "refresh_cache-d808804b-a2d0-4225-beb5-7d91e0cff3ac" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 783.279401] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 783.279580] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 783.279748] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 783.294541] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 783.295070] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg f0dd8e49c22845acb9c3f17ac3cd98ac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 783.301440] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f0dd8e49c22845acb9c3f17ac3cd98ac [ 783.363849] env[64020]: INFO nova.scheduler.client.report [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Deleted allocations for instance 8e8e1b50-fcae-41c6-8f5a-241692140cda [ 783.369431] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Expecting reply to msg 29000539fc804432ba978440a6e569d4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 783.380844] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 29000539fc804432ba978440a6e569d4 [ 783.555095] env[64020]: DEBUG nova.scheduler.client.report [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 783.557696] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 01c2652802b0443eb2e112ffd83d8144 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 783.572500] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 01c2652802b0443eb2e112ffd83d8144 [ 783.798257] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 783.798257] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 8ed285c0bc1541cd925ca2812daf356b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 783.806156] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ed285c0bc1541cd925ca2812daf356b [ 783.870872] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e1afa073-76a2-4fb9-a5ad-6b477a358a54 tempest-ListServersNegativeTestJSON-1814667196 tempest-ListServersNegativeTestJSON-1814667196-project-member] Lock "8e8e1b50-fcae-41c6-8f5a-241692140cda" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 153.730s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 783.871537] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 0a165eb2b0704293a13a7eefa262b2fc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 783.883089] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a165eb2b0704293a13a7eefa262b2fc [ 784.060143] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.887s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 784.061049] env[64020]: ERROR nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port fb363812-d8d1-49c5-95b7-693cacafc863, please check neutron logs for more information. [ 784.061049] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Traceback (most recent call last): [ 784.061049] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 784.061049] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] self.driver.spawn(context, instance, image_meta, [ 784.061049] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 784.061049] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 784.061049] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 784.061049] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] vm_ref = self.build_virtual_machine(instance, [ 784.061049] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 784.061049] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] vif_infos = vmwarevif.get_vif_info(self._session, [ 784.061049] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] for vif in network_info: [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] return self._sync_wrapper(fn, *args, **kwargs) [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] self.wait() [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] self[:] = self._gt.wait() [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] return self._exit_event.wait() [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] current.throw(*self._exc) [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 784.061501] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] result = function(*args, **kwargs) [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] return func(*args, **kwargs) [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] raise e [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] nwinfo = self.network_api.allocate_for_instance( [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] created_port_ids = self._update_ports_for_instance( [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] with excutils.save_and_reraise_exception(): [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] self.force_reraise() [ 784.061903] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 784.062346] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] raise self.value [ 784.062346] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 784.062346] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] updated_port = self._update_port( [ 784.062346] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 784.062346] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] _ensure_no_port_binding_failure(port) [ 784.062346] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 784.062346] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] raise exception.PortBindingFailed(port_id=port['id']) [ 784.062346] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] nova.exception.PortBindingFailed: Binding failed for port fb363812-d8d1-49c5-95b7-693cacafc863, please check neutron logs for more information. [ 784.062346] env[64020]: ERROR nova.compute.manager [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] [ 784.062346] env[64020]: DEBUG nova.compute.utils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Binding failed for port fb363812-d8d1-49c5-95b7-693cacafc863, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 784.064081] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.749s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 784.066148] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg e3cdc7935cc34de7a6581bb3ab61e360 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 784.067472] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Build of instance 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4 was re-scheduled: Binding failed for port fb363812-d8d1-49c5-95b7-693cacafc863, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 784.067994] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 784.068283] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "refresh_cache-2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 784.068453] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquired lock "refresh_cache-2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 784.068620] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 784.069188] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 2413732499264a7abc8d8e8c489958c5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 784.076448] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2413732499264a7abc8d8e8c489958c5 [ 784.100713] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3cdc7935cc34de7a6581bb3ab61e360 [ 784.301049] env[64020]: INFO nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: d808804b-a2d0-4225-beb5-7d91e0cff3ac] Took 1.02 seconds to deallocate network for instance. [ 784.303008] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg c39df604120d4f60a1ab6d08e43f724f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 784.335930] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c39df604120d4f60a1ab6d08e43f724f [ 784.377327] env[64020]: DEBUG nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 784.377327] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg ef1c156174314e7e8246a8493ca471a4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 784.407969] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef1c156174314e7e8246a8493ca471a4 [ 784.591424] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 784.679477] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 784.679990] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg b6d0f12af0a54a53be0bd0c0089c62f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 784.689384] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b6d0f12af0a54a53be0bd0c0089c62f5 [ 784.807032] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg b25b55eb0a91455e810b51d7889fc334 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 784.841735] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b25b55eb0a91455e810b51d7889fc334 [ 784.889751] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e281a51-606c-46fd-b49a-e477c7405206 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.897813] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 784.898784] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a795e0a9-9bfa-4a29-a139-ca53262e8671 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.929795] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6fedbcb-0d28-4a11-aa21-6bf68ec32efa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.937403] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc758f8a-c9d5-4614-a83c-e23c8951a839 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.953147] env[64020]: DEBUG nova.compute.provider_tree [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 784.953590] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 0f1009e0e16e46ef999f4feafb0de72b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 784.963008] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f1009e0e16e46ef999f4feafb0de72b [ 785.182770] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Releasing lock "refresh_cache-2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 785.183022] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 785.183206] env[64020]: DEBUG nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 785.183369] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 785.202837] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 785.203398] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 1941fca9040e43c3a953c041f382b135 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 785.210339] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1941fca9040e43c3a953c041f382b135 [ 785.337926] env[64020]: INFO nova.scheduler.client.report [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Deleted allocations for instance d808804b-a2d0-4225-beb5-7d91e0cff3ac [ 785.344042] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg c617ff4321524757aff55a94c8ac9772 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 785.362154] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c617ff4321524757aff55a94c8ac9772 [ 785.457508] env[64020]: DEBUG nova.scheduler.client.report [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 785.460159] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg dbc20d551b2942e2ad45f1d738e0cb12 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 785.470790] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dbc20d551b2942e2ad45f1d738e0cb12 [ 785.706718] env[64020]: DEBUG nova.network.neutron [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 785.707366] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 03aedfecaab14555b6c78d5199ad1ebb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 785.716148] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03aedfecaab14555b6c78d5199ad1ebb [ 785.846455] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "d808804b-a2d0-4225-beb5-7d91e0cff3ac" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 154.790s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 785.847427] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg b4eff3a461ba4ab8830efa1a1fcd9486 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 785.858973] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b4eff3a461ba4ab8830efa1a1fcd9486 [ 785.963354] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.900s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 785.963745] env[64020]: ERROR nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 260fff8f-aefb-4420-a11a-81a330ebc23c, please check neutron logs for more information. [ 785.963745] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Traceback (most recent call last): [ 785.963745] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 785.963745] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] self.driver.spawn(context, instance, image_meta, [ 785.963745] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 785.963745] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] self._vmops.spawn(context, instance, image_meta, injected_files, [ 785.963745] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 785.963745] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] vm_ref = self.build_virtual_machine(instance, [ 785.963745] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 785.963745] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] vif_infos = vmwarevif.get_vif_info(self._session, [ 785.963745] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] for vif in network_info: [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] return self._sync_wrapper(fn, *args, **kwargs) [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] self.wait() [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] self[:] = self._gt.wait() [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] return self._exit_event.wait() [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] current.throw(*self._exc) [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 785.964089] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] result = function(*args, **kwargs) [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] return func(*args, **kwargs) [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] raise e [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] nwinfo = self.network_api.allocate_for_instance( [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] created_port_ids = self._update_ports_for_instance( [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] with excutils.save_and_reraise_exception(): [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] self.force_reraise() [ 785.964600] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 785.965024] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] raise self.value [ 785.965024] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 785.965024] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] updated_port = self._update_port( [ 785.965024] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 785.965024] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] _ensure_no_port_binding_failure(port) [ 785.965024] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 785.965024] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] raise exception.PortBindingFailed(port_id=port['id']) [ 785.965024] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] nova.exception.PortBindingFailed: Binding failed for port 260fff8f-aefb-4420-a11a-81a330ebc23c, please check neutron logs for more information. [ 785.965024] env[64020]: ERROR nova.compute.manager [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] [ 785.965024] env[64020]: DEBUG nova.compute.utils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Binding failed for port 260fff8f-aefb-4420-a11a-81a330ebc23c, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 785.965586] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.269s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 785.967156] env[64020]: INFO nova.compute.claims [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 785.968900] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg c94ad6cf9e914553a37b6fd8568ec44c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 785.976480] env[64020]: DEBUG nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Build of instance 47516f85-60d8-4931-a205-b2ec1f7aec83 was re-scheduled: Binding failed for port 260fff8f-aefb-4420-a11a-81a330ebc23c, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 785.976896] env[64020]: DEBUG nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 785.977155] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Acquiring lock "refresh_cache-47516f85-60d8-4931-a205-b2ec1f7aec83" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 785.977301] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Acquired lock "refresh_cache-47516f85-60d8-4931-a205-b2ec1f7aec83" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 785.977475] env[64020]: DEBUG nova.network.neutron [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 785.977883] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 1a11444f7f60437db6fb318ef84cc846 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 785.987736] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a11444f7f60437db6fb318ef84cc846 [ 786.008100] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c94ad6cf9e914553a37b6fd8568ec44c [ 786.209943] env[64020]: INFO nova.compute.manager [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4] Took 1.03 seconds to deallocate network for instance. [ 786.211719] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg d1e7f4b6af0d424394e6f458ccbbc519 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 786.246974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1e7f4b6af0d424394e6f458ccbbc519 [ 786.349805] env[64020]: DEBUG nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 786.351527] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 233f33bfc8b446759d685a21f7feb4f7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 786.382135] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 233f33bfc8b446759d685a21f7feb4f7 [ 786.481226] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 23047107569a49f49d1a5435cc7764fa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 786.490309] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 23047107569a49f49d1a5435cc7764fa [ 786.498823] env[64020]: DEBUG nova.network.neutron [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 786.596478] env[64020]: DEBUG nova.network.neutron [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 786.596913] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 86c1f566064c4b00b33d838f30323953 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 786.605343] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86c1f566064c4b00b33d838f30323953 [ 786.716647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 10f473108bdd4eddbf9cb912cda87e04 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 786.753250] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10f473108bdd4eddbf9cb912cda87e04 [ 786.868879] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 787.101084] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Releasing lock "refresh_cache-47516f85-60d8-4931-a205-b2ec1f7aec83" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 787.101381] env[64020]: DEBUG nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 787.101531] env[64020]: DEBUG nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 787.101784] env[64020]: DEBUG nova.network.neutron [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 787.120339] env[64020]: DEBUG nova.network.neutron [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 787.120881] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 7971ee7780274b299b3767a17b8e3c92 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 787.131176] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7971ee7780274b299b3767a17b8e3c92 [ 787.172305] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquiring lock "7ba3e477-a789-4beb-90ac-651520836a5d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 787.172539] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "7ba3e477-a789-4beb-90ac-651520836a5d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 787.244559] env[64020]: INFO nova.scheduler.client.report [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Deleted allocations for instance 2e48bf2a-bdb4-4626-8e0a-c4556034a7b4 [ 787.254815] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg d7aac9c818c74693b74fcb2b48f6e134 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 787.263663] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7aac9c818c74693b74fcb2b48f6e134 [ 787.340377] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80e309ad-9d72-49e5-b93a-720671116d89 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.347793] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2e14117-391e-45ee-87bd-dbb1674242d6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.383950] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-443e9aa8-f3e9-4e66-8307-84f0090e3205 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.391614] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ec2a368-21d8-42ab-8532-01878bbf0f03 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.406748] env[64020]: DEBUG nova.compute.provider_tree [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 787.406748] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 444302e6b4784fa6bb1f0a7904400eaa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 787.419717] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 444302e6b4784fa6bb1f0a7904400eaa [ 787.490174] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 50eaa39ad38e48c59a3d0edec4441ace in queue reply_57893177120949e6a93cb88e15cd42b4 [ 787.506748] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50eaa39ad38e48c59a3d0edec4441ace [ 787.623424] env[64020]: DEBUG nova.network.neutron [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 787.624228] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 54580f209edc453fbf2c1fb1c06c5d15 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 787.632453] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54580f209edc453fbf2c1fb1c06c5d15 [ 787.752608] env[64020]: DEBUG oslo_concurrency.lockutils [None req-36e6f0df-c65c-49f9-bf44-1890e935119b tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "2e48bf2a-bdb4-4626-8e0a-c4556034a7b4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 156.665s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 787.753308] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 4f3cc906aee64b8eb1282aa26ce31d2f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 787.764992] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f3cc906aee64b8eb1282aa26ce31d2f [ 787.909443] env[64020]: DEBUG nova.scheduler.client.report [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 787.914708] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 95eba1669970468eb65fd41593fee125 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 787.931997] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95eba1669970468eb65fd41593fee125 [ 788.126964] env[64020]: INFO nova.compute.manager [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] [instance: 47516f85-60d8-4931-a205-b2ec1f7aec83] Took 1.03 seconds to deallocate network for instance. [ 788.128846] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg d2023b9e0afb4d78bab2e4deb5b4d6d1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 788.178983] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d2023b9e0afb4d78bab2e4deb5b4d6d1 [ 788.255535] env[64020]: DEBUG nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 788.257449] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg ff860b2eabe248b79de2ddd9f88821bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 788.299429] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff860b2eabe248b79de2ddd9f88821bc [ 788.415064] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.449s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 788.415598] env[64020]: DEBUG nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 788.417431] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 739bcd14018e41afb58cbdac25f74e90 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 788.418412] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.791s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 788.419815] env[64020]: INFO nova.compute.claims [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 788.421435] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 2dd889456d88494287ee7839999cc927 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 788.458840] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2dd889456d88494287ee7839999cc927 [ 788.459481] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 739bcd14018e41afb58cbdac25f74e90 [ 788.634100] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 6e35a811253b4da99eeb2e57c6a015ad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 788.669770] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6e35a811253b4da99eeb2e57c6a015ad [ 788.774345] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 788.924579] env[64020]: DEBUG nova.compute.utils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 788.925222] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 2b482eec69804ff298db7b12bbe61eb5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 788.927364] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 0465662cb112421abab97a1798cb2d64 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 788.928020] env[64020]: DEBUG nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 788.928020] env[64020]: DEBUG nova.network.neutron [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 788.938438] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0465662cb112421abab97a1798cb2d64 [ 788.939608] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b482eec69804ff298db7b12bbe61eb5 [ 788.955039] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 788.955250] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 788.973916] env[64020]: DEBUG nova.policy [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4d5dbd4d4e3243308747db3240e9a44a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '589de2dc1d2c4a6b933789878362e0e0', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 789.000496] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "f4017c56-63b3-420d-91ab-4565d82305fd" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 789.000717] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "f4017c56-63b3-420d-91ab-4565d82305fd" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 789.158813] env[64020]: INFO nova.scheduler.client.report [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Deleted allocations for instance 47516f85-60d8-4931-a205-b2ec1f7aec83 [ 789.164709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Expecting reply to msg 7423a636f313477d9a7826cab96e992d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 789.184565] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7423a636f313477d9a7826cab96e992d [ 789.279778] env[64020]: DEBUG nova.network.neutron [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Successfully created port: 04a04760-7eee-4ce9-98f4-56ed86f6375d {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 789.434097] env[64020]: DEBUG nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 789.436228] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 3554aec0ae7441dc97bddadb29b3a46a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 789.471827] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3554aec0ae7441dc97bddadb29b3a46a [ 789.674496] env[64020]: DEBUG oslo_concurrency.lockutils [None req-21337766-d685-4003-ab85-18b6a59091ab tempest-AttachInterfacesUnderV243Test-975117008 tempest-AttachInterfacesUnderV243Test-975117008-project-member] Lock "47516f85-60d8-4931-a205-b2ec1f7aec83" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 153.639s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 789.675127] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg e16ebf1b200a4957881714267999b194 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 789.687643] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e16ebf1b200a4957881714267999b194 [ 789.783176] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77d7cb4f-0f47-4003-9f52-e0f030279bdb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.791580] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-deaceb59-b2d2-42a5-bedf-5d5d4f314e32 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.824110] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81b4e99d-3b68-4e9b-b492-b8faa64ce0f2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.831960] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61d91ae2-4f88-414b-b21e-16ff82888a8e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.845559] env[64020]: DEBUG nova.compute.provider_tree [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 789.846208] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 7db3e1b61ff64a13bf20ebdc5827e162 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 789.853472] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7db3e1b61ff64a13bf20ebdc5827e162 [ 789.941615] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 23c91d8944d6431a8790d5733b45b385 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 789.948762] env[64020]: DEBUG nova.compute.manager [req-e67fb146-26d0-4290-a3fb-8590270c0a6b req-551d0f92-23af-4d9f-8e9b-f91ac789d52b service nova] [instance: f873c026-0fa1-4176-a30f-207767f87410] Received event network-changed-04a04760-7eee-4ce9-98f4-56ed86f6375d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 789.949023] env[64020]: DEBUG nova.compute.manager [req-e67fb146-26d0-4290-a3fb-8590270c0a6b req-551d0f92-23af-4d9f-8e9b-f91ac789d52b service nova] [instance: f873c026-0fa1-4176-a30f-207767f87410] Refreshing instance network info cache due to event network-changed-04a04760-7eee-4ce9-98f4-56ed86f6375d. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 789.949209] env[64020]: DEBUG oslo_concurrency.lockutils [req-e67fb146-26d0-4290-a3fb-8590270c0a6b req-551d0f92-23af-4d9f-8e9b-f91ac789d52b service nova] Acquiring lock "refresh_cache-f873c026-0fa1-4176-a30f-207767f87410" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 789.949376] env[64020]: DEBUG oslo_concurrency.lockutils [req-e67fb146-26d0-4290-a3fb-8590270c0a6b req-551d0f92-23af-4d9f-8e9b-f91ac789d52b service nova] Acquired lock "refresh_cache-f873c026-0fa1-4176-a30f-207767f87410" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 789.949525] env[64020]: DEBUG nova.network.neutron [req-e67fb146-26d0-4290-a3fb-8590270c0a6b req-551d0f92-23af-4d9f-8e9b-f91ac789d52b service nova] [instance: f873c026-0fa1-4176-a30f-207767f87410] Refreshing network info cache for port 04a04760-7eee-4ce9-98f4-56ed86f6375d {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 789.949953] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e67fb146-26d0-4290-a3fb-8590270c0a6b req-551d0f92-23af-4d9f-8e9b-f91ac789d52b service nova] Expecting reply to msg 6e7696207726443ab1542f96d77d5a20 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 789.957390] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6e7696207726443ab1542f96d77d5a20 [ 789.973958] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 23c91d8944d6431a8790d5733b45b385 [ 790.155876] env[64020]: ERROR nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 04a04760-7eee-4ce9-98f4-56ed86f6375d, please check neutron logs for more information. [ 790.155876] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 790.155876] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 790.155876] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 790.155876] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 790.155876] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 790.155876] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 790.155876] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 790.155876] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 790.155876] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 790.155876] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 790.155876] env[64020]: ERROR nova.compute.manager raise self.value [ 790.155876] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 790.155876] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 790.155876] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 790.155876] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 790.156703] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 790.156703] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 790.156703] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 04a04760-7eee-4ce9-98f4-56ed86f6375d, please check neutron logs for more information. [ 790.156703] env[64020]: ERROR nova.compute.manager [ 790.156703] env[64020]: Traceback (most recent call last): [ 790.156703] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 790.156703] env[64020]: listener.cb(fileno) [ 790.156703] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 790.156703] env[64020]: result = function(*args, **kwargs) [ 790.156703] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 790.156703] env[64020]: return func(*args, **kwargs) [ 790.156703] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 790.156703] env[64020]: raise e [ 790.156703] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 790.156703] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 790.156703] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 790.156703] env[64020]: created_port_ids = self._update_ports_for_instance( [ 790.156703] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 790.156703] env[64020]: with excutils.save_and_reraise_exception(): [ 790.156703] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 790.156703] env[64020]: self.force_reraise() [ 790.156703] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 790.156703] env[64020]: raise self.value [ 790.156703] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 790.156703] env[64020]: updated_port = self._update_port( [ 790.156703] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 790.156703] env[64020]: _ensure_no_port_binding_failure(port) [ 790.156703] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 790.156703] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 790.158065] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 04a04760-7eee-4ce9-98f4-56ed86f6375d, please check neutron logs for more information. [ 790.158065] env[64020]: Removing descriptor: 18 [ 790.177727] env[64020]: DEBUG nova.compute.manager [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 790.179795] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg f04fa18784014ad1b64783365ef1ccf1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 790.214490] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f04fa18784014ad1b64783365ef1ccf1 [ 790.348415] env[64020]: DEBUG nova.scheduler.client.report [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 790.350800] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 482b587ca3e244238cc498350d33c5f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 790.362746] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 482b587ca3e244238cc498350d33c5f5 [ 790.444782] env[64020]: DEBUG nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 790.470301] env[64020]: DEBUG nova.virt.hardware [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 790.470549] env[64020]: DEBUG nova.virt.hardware [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 790.470699] env[64020]: DEBUG nova.virt.hardware [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 790.470874] env[64020]: DEBUG nova.virt.hardware [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 790.471011] env[64020]: DEBUG nova.virt.hardware [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 790.471183] env[64020]: DEBUG nova.virt.hardware [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 790.471382] env[64020]: DEBUG nova.virt.hardware [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 790.471536] env[64020]: DEBUG nova.virt.hardware [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 790.471693] env[64020]: DEBUG nova.virt.hardware [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 790.471849] env[64020]: DEBUG nova.virt.hardware [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 790.472061] env[64020]: DEBUG nova.virt.hardware [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 790.472896] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a06ee828-243e-406a-926d-d64f6a99b096 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.476640] env[64020]: DEBUG nova.network.neutron [req-e67fb146-26d0-4290-a3fb-8590270c0a6b req-551d0f92-23af-4d9f-8e9b-f91ac789d52b service nova] [instance: f873c026-0fa1-4176-a30f-207767f87410] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 790.481630] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fcaf83d2-aa83-44ec-9b5b-ad02719e470f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.500396] env[64020]: ERROR nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 04a04760-7eee-4ce9-98f4-56ed86f6375d, please check neutron logs for more information. [ 790.500396] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] Traceback (most recent call last): [ 790.500396] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 790.500396] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] yield resources [ 790.500396] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 790.500396] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] self.driver.spawn(context, instance, image_meta, [ 790.500396] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 790.500396] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] self._vmops.spawn(context, instance, image_meta, injected_files, [ 790.500396] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 790.500396] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] vm_ref = self.build_virtual_machine(instance, [ 790.500396] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] vif_infos = vmwarevif.get_vif_info(self._session, [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] for vif in network_info: [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] return self._sync_wrapper(fn, *args, **kwargs) [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] self.wait() [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] self[:] = self._gt.wait() [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] return self._exit_event.wait() [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 790.500864] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] current.throw(*self._exc) [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] result = function(*args, **kwargs) [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] return func(*args, **kwargs) [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] raise e [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] nwinfo = self.network_api.allocate_for_instance( [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] created_port_ids = self._update_ports_for_instance( [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] with excutils.save_and_reraise_exception(): [ 790.501305] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 790.501710] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] self.force_reraise() [ 790.501710] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 790.501710] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] raise self.value [ 790.501710] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 790.501710] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] updated_port = self._update_port( [ 790.501710] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 790.501710] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] _ensure_no_port_binding_failure(port) [ 790.501710] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 790.501710] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] raise exception.PortBindingFailed(port_id=port['id']) [ 790.501710] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] nova.exception.PortBindingFailed: Binding failed for port 04a04760-7eee-4ce9-98f4-56ed86f6375d, please check neutron logs for more information. [ 790.501710] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] [ 790.501710] env[64020]: INFO nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Terminating instance [ 790.503467] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Acquiring lock "refresh_cache-f873c026-0fa1-4176-a30f-207767f87410" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 790.571325] env[64020]: DEBUG nova.network.neutron [req-e67fb146-26d0-4290-a3fb-8590270c0a6b req-551d0f92-23af-4d9f-8e9b-f91ac789d52b service nova] [instance: f873c026-0fa1-4176-a30f-207767f87410] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 790.571920] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e67fb146-26d0-4290-a3fb-8590270c0a6b req-551d0f92-23af-4d9f-8e9b-f91ac789d52b service nova] Expecting reply to msg 50e0b2a20eea4b2eb794a903be8758d6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 790.584535] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50e0b2a20eea4b2eb794a903be8758d6 [ 790.700381] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 790.853979] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.435s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 790.856467] env[64020]: DEBUG nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 790.856467] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 48dd56b5794b4dd2a9194d21c4e60025 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 790.857738] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.319s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 790.859160] env[64020]: INFO nova.compute.claims [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 790.860756] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 89a220234fb544b58ca6089ca5a46a82 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 790.889791] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 48dd56b5794b4dd2a9194d21c4e60025 [ 790.895450] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89a220234fb544b58ca6089ca5a46a82 [ 791.074709] env[64020]: DEBUG oslo_concurrency.lockutils [req-e67fb146-26d0-4290-a3fb-8590270c0a6b req-551d0f92-23af-4d9f-8e9b-f91ac789d52b service nova] Releasing lock "refresh_cache-f873c026-0fa1-4176-a30f-207767f87410" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 791.075133] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Acquired lock "refresh_cache-f873c026-0fa1-4176-a30f-207767f87410" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 791.075360] env[64020]: DEBUG nova.network.neutron [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 791.076160] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 6431ccfb852d4518a6f3bfe3b96bd2a6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 791.086627] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6431ccfb852d4518a6f3bfe3b96bd2a6 [ 791.364060] env[64020]: DEBUG nova.compute.utils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 791.369144] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg efdfca7030d9404f89be7887f7a8d240 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 791.369144] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg c911d40c0e2b4c7caf28d90f04ab3dc8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 791.370010] env[64020]: DEBUG nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 791.370218] env[64020]: DEBUG nova.network.neutron [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 791.377751] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c911d40c0e2b4c7caf28d90f04ab3dc8 [ 791.378225] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg efdfca7030d9404f89be7887f7a8d240 [ 791.444714] env[64020]: DEBUG nova.policy [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'eced18067cfa4884ab2176ab0433be63', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '11117df6330941bea05f374364f397ba', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 791.607107] env[64020]: DEBUG nova.network.neutron [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 791.749713] env[64020]: DEBUG nova.network.neutron [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Successfully created port: 02caced1-357c-4472-81cc-925703a5e48b {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 791.871211] env[64020]: DEBUG nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 791.873336] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 997756f66f124b03a735be42b784f4bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 791.874604] env[64020]: DEBUG nova.network.neutron [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 791.875008] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg ee40939107184a54b29b82f47e57a4ca in queue reply_57893177120949e6a93cb88e15cd42b4 [ 791.892631] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ee40939107184a54b29b82f47e57a4ca [ 791.908207] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 997756f66f124b03a735be42b784f4bc [ 792.061736] env[64020]: DEBUG nova.compute.manager [req-9802cc71-1153-46d5-aa33-d6d31f802b48 req-4d5f1b5c-3be6-47f3-9d4d-7749570b8491 service nova] [instance: f873c026-0fa1-4176-a30f-207767f87410] Received event network-vif-deleted-04a04760-7eee-4ce9-98f4-56ed86f6375d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 792.244986] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb7b93be-ef27-4eb1-ba78-1628a8f3cc50 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.256116] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d9f243f-a7da-40ee-a969-3727ccd2fab3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.282894] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1af4143e-5772-49c7-889b-2762f1fbc245 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.290061] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d303f579-aaa1-4a71-96a2-b5874269529d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.306356] env[64020]: DEBUG nova.compute.provider_tree [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 792.306868] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg ed2dfda8da03454696648671bbc44cfc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 792.314356] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed2dfda8da03454696648671bbc44cfc [ 792.386854] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Releasing lock "refresh_cache-f873c026-0fa1-4176-a30f-207767f87410" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 792.386854] env[64020]: DEBUG nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 792.387130] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 792.389579] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 0f3ebeee58ec4e78826a294cd9791c88 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 792.390482] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-83938311-d251-46c4-b3aa-d49c8f8fec5e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.399349] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7703faaa-c51f-40e7-885f-3342fd6dd6ce {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.421832] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f873c026-0fa1-4176-a30f-207767f87410 could not be found. [ 792.422059] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 792.422261] env[64020]: INFO nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Took 0.04 seconds to destroy the instance on the hypervisor. [ 792.422529] env[64020]: DEBUG oslo.service.loopingcall [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 792.422880] env[64020]: DEBUG nova.compute.manager [-] [instance: f873c026-0fa1-4176-a30f-207767f87410] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 792.422981] env[64020]: DEBUG nova.network.neutron [-] [instance: f873c026-0fa1-4176-a30f-207767f87410] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 792.428881] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f3ebeee58ec4e78826a294cd9791c88 [ 792.437333] env[64020]: DEBUG nova.network.neutron [-] [instance: f873c026-0fa1-4176-a30f-207767f87410] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 792.437847] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5ee17aa04fd74dc5a4374f8a5cfdf041 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 792.445386] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ee17aa04fd74dc5a4374f8a5cfdf041 [ 792.696254] env[64020]: ERROR nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 02caced1-357c-4472-81cc-925703a5e48b, please check neutron logs for more information. [ 792.696254] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 792.696254] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 792.696254] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 792.696254] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 792.696254] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 792.696254] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 792.696254] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 792.696254] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 792.696254] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 792.696254] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 792.696254] env[64020]: ERROR nova.compute.manager raise self.value [ 792.696254] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 792.696254] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 792.696254] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 792.696254] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 792.696757] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 792.696757] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 792.696757] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 02caced1-357c-4472-81cc-925703a5e48b, please check neutron logs for more information. [ 792.696757] env[64020]: ERROR nova.compute.manager [ 792.696757] env[64020]: Traceback (most recent call last): [ 792.696757] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 792.696757] env[64020]: listener.cb(fileno) [ 792.696757] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 792.696757] env[64020]: result = function(*args, **kwargs) [ 792.696757] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 792.696757] env[64020]: return func(*args, **kwargs) [ 792.696757] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 792.696757] env[64020]: raise e [ 792.696757] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 792.696757] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 792.696757] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 792.696757] env[64020]: created_port_ids = self._update_ports_for_instance( [ 792.696757] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 792.696757] env[64020]: with excutils.save_and_reraise_exception(): [ 792.696757] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 792.696757] env[64020]: self.force_reraise() [ 792.696757] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 792.696757] env[64020]: raise self.value [ 792.696757] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 792.696757] env[64020]: updated_port = self._update_port( [ 792.696757] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 792.696757] env[64020]: _ensure_no_port_binding_failure(port) [ 792.696757] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 792.696757] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 792.697566] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 02caced1-357c-4472-81cc-925703a5e48b, please check neutron logs for more information. [ 792.697566] env[64020]: Removing descriptor: 18 [ 792.810185] env[64020]: DEBUG nova.scheduler.client.report [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 792.812729] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 850bfa664a5e448385facb781c2a4307 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 792.829598] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 850bfa664a5e448385facb781c2a4307 [ 792.895381] env[64020]: DEBUG nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 792.924831] env[64020]: DEBUG nova.virt.hardware [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 792.925537] env[64020]: DEBUG nova.virt.hardware [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 792.925776] env[64020]: DEBUG nova.virt.hardware [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 792.925974] env[64020]: DEBUG nova.virt.hardware [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 792.926122] env[64020]: DEBUG nova.virt.hardware [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 792.926269] env[64020]: DEBUG nova.virt.hardware [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 792.926478] env[64020]: DEBUG nova.virt.hardware [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 792.926632] env[64020]: DEBUG nova.virt.hardware [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 792.926800] env[64020]: DEBUG nova.virt.hardware [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 792.926950] env[64020]: DEBUG nova.virt.hardware [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 792.927112] env[64020]: DEBUG nova.virt.hardware [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 792.927964] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c953f48e-b6bb-4ad7-8d71-696831f85bd2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.938465] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-622f2777-0ec8-4274-8ec5-fab36545c015 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.942046] env[64020]: DEBUG nova.network.neutron [-] [instance: f873c026-0fa1-4176-a30f-207767f87410] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 792.942653] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg fb3ea711473145c293b32ddd1334be7b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 792.954511] env[64020]: ERROR nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 02caced1-357c-4472-81cc-925703a5e48b, please check neutron logs for more information. [ 792.954511] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Traceback (most recent call last): [ 792.954511] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 792.954511] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] yield resources [ 792.954511] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 792.954511] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] self.driver.spawn(context, instance, image_meta, [ 792.954511] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 792.954511] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 792.954511] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 792.954511] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] vm_ref = self.build_virtual_machine(instance, [ 792.954511] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] vif_infos = vmwarevif.get_vif_info(self._session, [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] for vif in network_info: [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] return self._sync_wrapper(fn, *args, **kwargs) [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] self.wait() [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] self[:] = self._gt.wait() [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] return self._exit_event.wait() [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 792.954871] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] current.throw(*self._exc) [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] result = function(*args, **kwargs) [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] return func(*args, **kwargs) [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] raise e [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] nwinfo = self.network_api.allocate_for_instance( [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] created_port_ids = self._update_ports_for_instance( [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] with excutils.save_and_reraise_exception(): [ 792.955227] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 792.955579] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] self.force_reraise() [ 792.955579] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 792.955579] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] raise self.value [ 792.955579] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 792.955579] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] updated_port = self._update_port( [ 792.955579] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 792.955579] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] _ensure_no_port_binding_failure(port) [ 792.955579] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 792.955579] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] raise exception.PortBindingFailed(port_id=port['id']) [ 792.955579] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] nova.exception.PortBindingFailed: Binding failed for port 02caced1-357c-4472-81cc-925703a5e48b, please check neutron logs for more information. [ 792.955579] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] [ 792.955579] env[64020]: INFO nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Terminating instance [ 792.956895] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb3ea711473145c293b32ddd1334be7b [ 792.957519] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Acquiring lock "refresh_cache-052ae9f0-bb03-4747-b8f9-708e4a7c147c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 792.957696] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Acquired lock "refresh_cache-052ae9f0-bb03-4747-b8f9-708e4a7c147c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 792.957863] env[64020]: DEBUG nova.network.neutron [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 792.958358] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 79e52b80c9be4bc5ab8be25fcaf0a33b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 792.965849] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 79e52b80c9be4bc5ab8be25fcaf0a33b [ 793.315700] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.458s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 793.316124] env[64020]: DEBUG nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 793.317892] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 19f650615d6148fd95ffd69e3500413a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 793.318947] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.766s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 793.320344] env[64020]: INFO nova.compute.claims [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 793.321805] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 66df68480db64b82a789cd77af7e1494 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 793.365351] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19f650615d6148fd95ffd69e3500413a [ 793.365957] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66df68480db64b82a789cd77af7e1494 [ 793.445247] env[64020]: INFO nova.compute.manager [-] [instance: f873c026-0fa1-4176-a30f-207767f87410] Took 1.02 seconds to deallocate network for instance. [ 793.447802] env[64020]: DEBUG nova.compute.claims [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 793.447975] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 793.474741] env[64020]: DEBUG nova.network.neutron [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 793.564235] env[64020]: DEBUG nova.network.neutron [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 793.564755] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 3fbf5efa2cfe4cf49714088edabfbe96 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 793.575899] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3fbf5efa2cfe4cf49714088edabfbe96 [ 793.826846] env[64020]: DEBUG nova.compute.utils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 793.827490] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 9fadc46b30734b1d98e81c8bbccb52fb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 793.832031] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 5d528690ce5e4b34bb90a2fbd484dad2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 793.832031] env[64020]: DEBUG nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 793.832031] env[64020]: DEBUG nova.network.neutron [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 793.836910] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d528690ce5e4b34bb90a2fbd484dad2 [ 793.841330] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9fadc46b30734b1d98e81c8bbccb52fb [ 793.899336] env[64020]: DEBUG nova.policy [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c725ab9b1dfa404db080a31531fbcb0c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'feab00fde0d8432b9e34ced0f592dd98', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 794.067488] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Releasing lock "refresh_cache-052ae9f0-bb03-4747-b8f9-708e4a7c147c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 794.067930] env[64020]: DEBUG nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 794.068134] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 794.068494] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-35d35322-89a7-43d7-8263-2d366990d046 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.077422] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-01d71ed7-ea9e-44af-b1ce-ee6f321ff22b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.105959] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 052ae9f0-bb03-4747-b8f9-708e4a7c147c could not be found. [ 794.106187] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 794.106394] env[64020]: INFO nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Took 0.04 seconds to destroy the instance on the hypervisor. [ 794.106646] env[64020]: DEBUG oslo.service.loopingcall [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 794.106870] env[64020]: DEBUG nova.compute.manager [-] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 794.106960] env[64020]: DEBUG nova.network.neutron [-] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 794.123215] env[64020]: DEBUG nova.compute.manager [req-04e45eeb-051f-4272-845d-477a0a549aec req-d218908b-c963-4c9a-80a0-73ab7acb17ef service nova] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Received event network-changed-02caced1-357c-4472-81cc-925703a5e48b {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 794.123442] env[64020]: DEBUG nova.compute.manager [req-04e45eeb-051f-4272-845d-477a0a549aec req-d218908b-c963-4c9a-80a0-73ab7acb17ef service nova] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Refreshing instance network info cache due to event network-changed-02caced1-357c-4472-81cc-925703a5e48b. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 794.123767] env[64020]: DEBUG oslo_concurrency.lockutils [req-04e45eeb-051f-4272-845d-477a0a549aec req-d218908b-c963-4c9a-80a0-73ab7acb17ef service nova] Acquiring lock "refresh_cache-052ae9f0-bb03-4747-b8f9-708e4a7c147c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 794.124069] env[64020]: DEBUG oslo_concurrency.lockutils [req-04e45eeb-051f-4272-845d-477a0a549aec req-d218908b-c963-4c9a-80a0-73ab7acb17ef service nova] Acquired lock "refresh_cache-052ae9f0-bb03-4747-b8f9-708e4a7c147c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 794.124153] env[64020]: DEBUG nova.network.neutron [req-04e45eeb-051f-4272-845d-477a0a549aec req-d218908b-c963-4c9a-80a0-73ab7acb17ef service nova] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Refreshing network info cache for port 02caced1-357c-4472-81cc-925703a5e48b {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 794.124622] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-04e45eeb-051f-4272-845d-477a0a549aec req-d218908b-c963-4c9a-80a0-73ab7acb17ef service nova] Expecting reply to msg 4064c0543ff14e24825b9a17a61ef602 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 794.125892] env[64020]: DEBUG nova.network.neutron [-] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 794.126373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c227118d1b234d16a29a0257f6d0f357 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 794.131043] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4064c0543ff14e24825b9a17a61ef602 [ 794.132787] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c227118d1b234d16a29a0257f6d0f357 [ 794.331465] env[64020]: DEBUG nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 794.333336] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 751df0c9f64d424795c6665aece5e5a4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 794.380484] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 751df0c9f64d424795c6665aece5e5a4 [ 794.588400] env[64020]: DEBUG nova.network.neutron [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Successfully created port: 417cecbc-3d8a-4119-b90d-38716e59bc58 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 794.630969] env[64020]: DEBUG nova.network.neutron [-] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 794.631430] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 45de74297048463db336b47ee501f363 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 794.639461] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 45de74297048463db336b47ee501f363 [ 794.646856] env[64020]: DEBUG nova.network.neutron [req-04e45eeb-051f-4272-845d-477a0a549aec req-d218908b-c963-4c9a-80a0-73ab7acb17ef service nova] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 794.669816] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-166fb786-142b-429d-95f5-8f1069086d26 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.678221] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec755a3a-d6ae-4f6b-acc3-429564bbb845 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.711339] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-215850bc-1513-4073-9b1d-2ea201c42374 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.718660] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-450c8691-6e9e-441c-91d9-1a7faf286de7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.731360] env[64020]: DEBUG nova.compute.provider_tree [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 794.731867] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg aafcf9b8d52140f784db57b6ec414f4b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 794.739710] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aafcf9b8d52140f784db57b6ec414f4b [ 794.748168] env[64020]: DEBUG nova.network.neutron [req-04e45eeb-051f-4272-845d-477a0a549aec req-d218908b-c963-4c9a-80a0-73ab7acb17ef service nova] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 794.748659] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-04e45eeb-051f-4272-845d-477a0a549aec req-d218908b-c963-4c9a-80a0-73ab7acb17ef service nova] Expecting reply to msg 9459d70ac4dc4451b1c14a737e659fce in queue reply_57893177120949e6a93cb88e15cd42b4 [ 794.756042] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9459d70ac4dc4451b1c14a737e659fce [ 794.839361] env[64020]: INFO nova.virt.block_device [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Booting with volume c9dd565d-1309-40cd-8dfd-a7b81694a1ea at /dev/sda [ 794.904993] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-098e9a98-60b1-4b47-a277-b118afa91278 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.911475] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e8898f3-0937-43ef-8868-0f1996e4ba08 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.935657] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1452a7dc-c63e-4623-b0c0-f568ab2f94a4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.946124] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6991575b-794f-41d7-b62e-4057455697bd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.979034] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea63283f-09f6-4b22-a79a-0b18bec8152c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.985745] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd30190b-1465-4ffd-a69d-ad500752800e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.010526] env[64020]: DEBUG nova.virt.block_device [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Updating existing volume attachment record: be600fa0-8fe9-41e1-ab74-1b225f72a751 {{(pid=64020) _volume_attach /opt/stack/nova/nova/virt/block_device.py:665}} [ 795.133465] env[64020]: INFO nova.compute.manager [-] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Took 1.03 seconds to deallocate network for instance. [ 795.136200] env[64020]: DEBUG nova.compute.claims [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 795.136624] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 795.238278] env[64020]: DEBUG nova.scheduler.client.report [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 795.238278] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg a34e2f17ea3c49d6870e82ef44ff67c6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 795.251034] env[64020]: DEBUG oslo_concurrency.lockutils [req-04e45eeb-051f-4272-845d-477a0a549aec req-d218908b-c963-4c9a-80a0-73ab7acb17ef service nova] Releasing lock "refresh_cache-052ae9f0-bb03-4747-b8f9-708e4a7c147c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 795.251034] env[64020]: DEBUG nova.compute.manager [req-04e45eeb-051f-4272-845d-477a0a549aec req-d218908b-c963-4c9a-80a0-73ab7acb17ef service nova] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Received event network-vif-deleted-02caced1-357c-4472-81cc-925703a5e48b {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 795.255274] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a34e2f17ea3c49d6870e82ef44ff67c6 [ 795.573376] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg ee5f27f7b30d40219cb3c272bfb29a63 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 795.584221] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ee5f27f7b30d40219cb3c272bfb29a63 [ 795.740175] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.421s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 795.741511] env[64020]: DEBUG nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 795.742406] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 45ed2bc4da28487ea652fcef0c7d7ae0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 795.743356] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.240s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 795.745199] env[64020]: INFO nova.compute.claims [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 795.750186] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg c40bc6742bc745d4b55486532bae7df7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 795.750186] env[64020]: ERROR nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 417cecbc-3d8a-4119-b90d-38716e59bc58, please check neutron logs for more information. [ 795.750186] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 795.750186] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 795.750186] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 795.750186] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 795.750186] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 795.750186] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 795.750186] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 795.750186] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 795.750186] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 795.750186] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 795.750186] env[64020]: ERROR nova.compute.manager raise self.value [ 795.750186] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 795.750839] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 795.750839] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 795.750839] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 795.750839] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 795.750839] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 795.750839] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 417cecbc-3d8a-4119-b90d-38716e59bc58, please check neutron logs for more information. [ 795.750839] env[64020]: ERROR nova.compute.manager [ 795.750839] env[64020]: Traceback (most recent call last): [ 795.750839] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 795.750839] env[64020]: listener.cb(fileno) [ 795.750839] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 795.750839] env[64020]: result = function(*args, **kwargs) [ 795.750839] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 795.750839] env[64020]: return func(*args, **kwargs) [ 795.750839] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 795.750839] env[64020]: raise e [ 795.750839] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 795.750839] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 795.750839] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 795.750839] env[64020]: created_port_ids = self._update_ports_for_instance( [ 795.750839] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 795.750839] env[64020]: with excutils.save_and_reraise_exception(): [ 795.750839] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 795.750839] env[64020]: self.force_reraise() [ 795.750839] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 795.750839] env[64020]: raise self.value [ 795.750839] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 795.750839] env[64020]: updated_port = self._update_port( [ 795.751791] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 795.751791] env[64020]: _ensure_no_port_binding_failure(port) [ 795.751791] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 795.751791] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 795.751791] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 417cecbc-3d8a-4119-b90d-38716e59bc58, please check neutron logs for more information. [ 795.751791] env[64020]: Removing descriptor: 18 [ 795.809443] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 45ed2bc4da28487ea652fcef0c7d7ae0 [ 795.812408] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c40bc6742bc745d4b55486532bae7df7 [ 795.884570] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 795.884794] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 795.885499] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg f7d1cc86b4a54194b41785ad572c8610 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 795.899006] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7d1cc86b4a54194b41785ad572c8610 [ 796.113647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 59f5828a7ae540ebbe7fee138e694883 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 796.125534] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59f5828a7ae540ebbe7fee138e694883 [ 796.156163] env[64020]: DEBUG nova.compute.manager [req-691878f2-a76a-4493-bb78-2e35d2b7e4f1 req-2cb33c7d-db8e-42b1-8b77-6dfc7e4d59cd service nova] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Received event network-changed-417cecbc-3d8a-4119-b90d-38716e59bc58 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 796.156423] env[64020]: DEBUG nova.compute.manager [req-691878f2-a76a-4493-bb78-2e35d2b7e4f1 req-2cb33c7d-db8e-42b1-8b77-6dfc7e4d59cd service nova] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Refreshing instance network info cache due to event network-changed-417cecbc-3d8a-4119-b90d-38716e59bc58. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 796.156554] env[64020]: DEBUG oslo_concurrency.lockutils [req-691878f2-a76a-4493-bb78-2e35d2b7e4f1 req-2cb33c7d-db8e-42b1-8b77-6dfc7e4d59cd service nova] Acquiring lock "refresh_cache-b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 796.156689] env[64020]: DEBUG oslo_concurrency.lockutils [req-691878f2-a76a-4493-bb78-2e35d2b7e4f1 req-2cb33c7d-db8e-42b1-8b77-6dfc7e4d59cd service nova] Acquired lock "refresh_cache-b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 796.156837] env[64020]: DEBUG nova.network.neutron [req-691878f2-a76a-4493-bb78-2e35d2b7e4f1 req-2cb33c7d-db8e-42b1-8b77-6dfc7e4d59cd service nova] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Refreshing network info cache for port 417cecbc-3d8a-4119-b90d-38716e59bc58 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 796.157249] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-691878f2-a76a-4493-bb78-2e35d2b7e4f1 req-2cb33c7d-db8e-42b1-8b77-6dfc7e4d59cd service nova] Expecting reply to msg 6502a8f17efd483f95e543b3ac5947ac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 796.163241] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6502a8f17efd483f95e543b3ac5947ac [ 796.249982] env[64020]: DEBUG nova.compute.utils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 796.250625] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 133176e72fea47c79115a882df566846 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 796.253079] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg d2e0b775ab8f43e5a3277065231bc903 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 796.253836] env[64020]: DEBUG nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 796.253997] env[64020]: DEBUG nova.network.neutron [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 796.260269] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d2e0b775ab8f43e5a3277065231bc903 [ 796.268121] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 133176e72fea47c79115a882df566846 [ 796.314550] env[64020]: DEBUG nova.policy [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2e95ca7785f34fd39b0c50374c7debb6', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e0068a323fd04f47b7ae69d1d68a04c0', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 796.383812] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Acquiring lock "5a021861-9784-431d-b717-c7b24fe8525c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.384053] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Lock "5a021861-9784-431d-b717-c7b24fe8525c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.389913] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 796.390166] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Starting heal instance info cache {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 796.390232] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Rebuilding the list of instances to heal {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 796.390690] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 36d160481df64cd68af24b18b1105a6c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 796.412948] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36d160481df64cd68af24b18b1105a6c [ 796.573683] env[64020]: DEBUG nova.network.neutron [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Successfully created port: 4dba8130-7cfd-4940-86fb-303a6f32e278 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 796.618064] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 7d2cec16173d4a058c7273c759c444a4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 796.657155] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d2cec16173d4a058c7273c759c444a4 [ 796.672765] env[64020]: DEBUG nova.network.neutron [req-691878f2-a76a-4493-bb78-2e35d2b7e4f1 req-2cb33c7d-db8e-42b1-8b77-6dfc7e4d59cd service nova] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 796.750966] env[64020]: DEBUG nova.network.neutron [req-691878f2-a76a-4493-bb78-2e35d2b7e4f1 req-2cb33c7d-db8e-42b1-8b77-6dfc7e4d59cd service nova] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 796.751565] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-691878f2-a76a-4493-bb78-2e35d2b7e4f1 req-2cb33c7d-db8e-42b1-8b77-6dfc7e4d59cd service nova] Expecting reply to msg b97a55729ee147ab8facf89312938dd2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 796.757215] env[64020]: DEBUG nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 796.758837] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 27529f0bbb6f4d6bb4603dea16870de7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 796.762684] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b97a55729ee147ab8facf89312938dd2 [ 796.817078] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27529f0bbb6f4d6bb4603dea16870de7 [ 796.894002] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 796.894217] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: f873c026-0fa1-4176-a30f-207767f87410] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 796.894381] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 796.894534] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 796.894680] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 1954e745-783e-417e-aafc-265da55066cf] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 796.894827] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 796.894973] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Didn't find any instances for network info cache update. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 796.895187] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 796.895375] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 796.895551] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 796.895725] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 796.895894] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 796.896091] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 796.896246] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=64020) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 796.896421] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager.update_available_resource {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 796.896789] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 557eb0e688264f78921b64eb2c6fa142 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 796.911172] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 557eb0e688264f78921b64eb2c6fa142 [ 797.122735] env[64020]: DEBUG nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 797.123272] env[64020]: DEBUG nova.virt.hardware [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 797.123485] env[64020]: DEBUG nova.virt.hardware [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 797.124177] env[64020]: DEBUG nova.virt.hardware [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 797.124177] env[64020]: DEBUG nova.virt.hardware [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 797.124177] env[64020]: DEBUG nova.virt.hardware [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 797.124177] env[64020]: DEBUG nova.virt.hardware [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 797.124392] env[64020]: DEBUG nova.virt.hardware [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 797.124431] env[64020]: DEBUG nova.virt.hardware [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 797.124566] env[64020]: DEBUG nova.virt.hardware [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 797.124724] env[64020]: DEBUG nova.virt.hardware [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 797.124889] env[64020]: DEBUG nova.virt.hardware [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 797.129108] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3265d44e-c5a1-4db2-bae3-1a8dd46e010a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.138485] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ce2d65e-a727-48b0-bd3b-aa4c9f972982 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.157257] env[64020]: ERROR nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 417cecbc-3d8a-4119-b90d-38716e59bc58, please check neutron logs for more information. [ 797.157257] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Traceback (most recent call last): [ 797.157257] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 797.157257] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] yield resources [ 797.157257] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 797.157257] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] self.driver.spawn(context, instance, image_meta, [ 797.157257] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 797.157257] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 797.157257] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 797.157257] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] vm_ref = self.build_virtual_machine(instance, [ 797.157257] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] vif_infos = vmwarevif.get_vif_info(self._session, [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] for vif in network_info: [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] return self._sync_wrapper(fn, *args, **kwargs) [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] self.wait() [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] self[:] = self._gt.wait() [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] return self._exit_event.wait() [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 797.157660] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] current.throw(*self._exc) [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] result = function(*args, **kwargs) [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] return func(*args, **kwargs) [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] raise e [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] nwinfo = self.network_api.allocate_for_instance( [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] created_port_ids = self._update_ports_for_instance( [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] with excutils.save_and_reraise_exception(): [ 797.158082] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 797.158501] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] self.force_reraise() [ 797.158501] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 797.158501] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] raise self.value [ 797.158501] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 797.158501] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] updated_port = self._update_port( [ 797.158501] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 797.158501] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] _ensure_no_port_binding_failure(port) [ 797.158501] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 797.158501] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] raise exception.PortBindingFailed(port_id=port['id']) [ 797.158501] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] nova.exception.PortBindingFailed: Binding failed for port 417cecbc-3d8a-4119-b90d-38716e59bc58, please check neutron logs for more information. [ 797.158501] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] [ 797.158501] env[64020]: INFO nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Terminating instance [ 797.161549] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Acquiring lock "refresh_cache-b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 797.173308] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-862b67c0-1736-4d72-8df4-3a3ac686333b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.180225] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-01409dfd-479b-4ad0-8554-8df55361eac2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.210290] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a40ba0c-645d-4682-80e4-95add227e69f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.218125] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48b537ce-f426-430f-9869-5a7f7056b3f4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.230901] env[64020]: DEBUG nova.compute.provider_tree [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 797.232709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 7869d39dc9564fc79025b75195474395 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 797.241524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7869d39dc9564fc79025b75195474395 [ 797.253450] env[64020]: DEBUG oslo_concurrency.lockutils [req-691878f2-a76a-4493-bb78-2e35d2b7e4f1 req-2cb33c7d-db8e-42b1-8b77-6dfc7e4d59cd service nova] Releasing lock "refresh_cache-b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 797.253723] env[64020]: DEBUG nova.compute.manager [req-691878f2-a76a-4493-bb78-2e35d2b7e4f1 req-2cb33c7d-db8e-42b1-8b77-6dfc7e4d59cd service nova] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Received event network-vif-deleted-417cecbc-3d8a-4119-b90d-38716e59bc58 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 797.254065] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Acquired lock "refresh_cache-b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 797.254223] env[64020]: DEBUG nova.network.neutron [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 797.254663] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg e512fb007051477db77395e1b676958e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 797.261172] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e512fb007051477db77395e1b676958e [ 797.264896] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 06eeb57a42e2450a9b45373c99d4f307 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 797.295022] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06eeb57a42e2450a9b45373c99d4f307 [ 797.399671] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 797.503166] env[64020]: ERROR nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4dba8130-7cfd-4940-86fb-303a6f32e278, please check neutron logs for more information. [ 797.503166] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 797.503166] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 797.503166] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 797.503166] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 797.503166] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 797.503166] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 797.503166] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 797.503166] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 797.503166] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 797.503166] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 797.503166] env[64020]: ERROR nova.compute.manager raise self.value [ 797.503166] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 797.503166] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 797.503166] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 797.503166] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 797.503694] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 797.503694] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 797.503694] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4dba8130-7cfd-4940-86fb-303a6f32e278, please check neutron logs for more information. [ 797.503694] env[64020]: ERROR nova.compute.manager [ 797.503694] env[64020]: Traceback (most recent call last): [ 797.503694] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 797.503694] env[64020]: listener.cb(fileno) [ 797.503694] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 797.503694] env[64020]: result = function(*args, **kwargs) [ 797.503694] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 797.503694] env[64020]: return func(*args, **kwargs) [ 797.503694] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 797.503694] env[64020]: raise e [ 797.503694] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 797.503694] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 797.503694] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 797.503694] env[64020]: created_port_ids = self._update_ports_for_instance( [ 797.503694] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 797.503694] env[64020]: with excutils.save_and_reraise_exception(): [ 797.503694] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 797.503694] env[64020]: self.force_reraise() [ 797.503694] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 797.503694] env[64020]: raise self.value [ 797.503694] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 797.503694] env[64020]: updated_port = self._update_port( [ 797.503694] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 797.503694] env[64020]: _ensure_no_port_binding_failure(port) [ 797.503694] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 797.503694] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 797.504693] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 4dba8130-7cfd-4940-86fb-303a6f32e278, please check neutron logs for more information. [ 797.504693] env[64020]: Removing descriptor: 18 [ 797.736056] env[64020]: DEBUG nova.scheduler.client.report [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 797.736960] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 1606e875ce9249f8823d54f972587f08 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 797.747763] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1606e875ce9249f8823d54f972587f08 [ 797.767770] env[64020]: DEBUG nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 797.771330] env[64020]: DEBUG nova.network.neutron [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 797.791349] env[64020]: DEBUG nova.virt.hardware [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 797.791588] env[64020]: DEBUG nova.virt.hardware [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 797.791742] env[64020]: DEBUG nova.virt.hardware [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 797.791918] env[64020]: DEBUG nova.virt.hardware [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 797.792070] env[64020]: DEBUG nova.virt.hardware [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 797.792414] env[64020]: DEBUG nova.virt.hardware [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 797.792626] env[64020]: DEBUG nova.virt.hardware [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 797.792781] env[64020]: DEBUG nova.virt.hardware [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 797.792939] env[64020]: DEBUG nova.virt.hardware [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 797.793094] env[64020]: DEBUG nova.virt.hardware [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 797.793258] env[64020]: DEBUG nova.virt.hardware [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 797.794127] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a68bb595-e74d-4bad-89d5-b1c70975a6ae {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.801506] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6cfffc15-e075-4565-8759-e622307cd0b2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.814786] env[64020]: ERROR nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4dba8130-7cfd-4940-86fb-303a6f32e278, please check neutron logs for more information. [ 797.814786] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] Traceback (most recent call last): [ 797.814786] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 797.814786] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] yield resources [ 797.814786] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 797.814786] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] self.driver.spawn(context, instance, image_meta, [ 797.814786] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 797.814786] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] self._vmops.spawn(context, instance, image_meta, injected_files, [ 797.814786] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 797.814786] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] vm_ref = self.build_virtual_machine(instance, [ 797.814786] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] vif_infos = vmwarevif.get_vif_info(self._session, [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] for vif in network_info: [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] return self._sync_wrapper(fn, *args, **kwargs) [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] self.wait() [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] self[:] = self._gt.wait() [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] return self._exit_event.wait() [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 797.815156] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] current.throw(*self._exc) [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] result = function(*args, **kwargs) [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] return func(*args, **kwargs) [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] raise e [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] nwinfo = self.network_api.allocate_for_instance( [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] created_port_ids = self._update_ports_for_instance( [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] with excutils.save_and_reraise_exception(): [ 797.815513] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 797.815876] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] self.force_reraise() [ 797.815876] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 797.815876] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] raise self.value [ 797.815876] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 797.815876] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] updated_port = self._update_port( [ 797.815876] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 797.815876] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] _ensure_no_port_binding_failure(port) [ 797.815876] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 797.815876] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] raise exception.PortBindingFailed(port_id=port['id']) [ 797.815876] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] nova.exception.PortBindingFailed: Binding failed for port 4dba8130-7cfd-4940-86fb-303a6f32e278, please check neutron logs for more information. [ 797.815876] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] [ 797.815876] env[64020]: INFO nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Terminating instance [ 797.819206] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Acquiring lock "refresh_cache-1954e745-783e-417e-aafc-265da55066cf" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 797.819371] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Acquired lock "refresh_cache-1954e745-783e-417e-aafc-265da55066cf" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 797.819538] env[64020]: DEBUG nova.network.neutron [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 797.819944] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg bc85d3fc275a4e8197ef75d402e41718 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 797.826745] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc85d3fc275a4e8197ef75d402e41718 [ 797.857226] env[64020]: DEBUG nova.network.neutron [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 797.857759] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 40dddebcbbc54d4b977186c5cdaef8db in queue reply_57893177120949e6a93cb88e15cd42b4 [ 797.865551] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40dddebcbbc54d4b977186c5cdaef8db [ 798.181292] env[64020]: DEBUG nova.compute.manager [req-2feb2b42-994d-46c0-b4bb-ecf481fbf7ad req-98c0c86d-925e-4f51-9e06-909b8d211634 service nova] [instance: 1954e745-783e-417e-aafc-265da55066cf] Received event network-changed-4dba8130-7cfd-4940-86fb-303a6f32e278 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 798.181448] env[64020]: DEBUG nova.compute.manager [req-2feb2b42-994d-46c0-b4bb-ecf481fbf7ad req-98c0c86d-925e-4f51-9e06-909b8d211634 service nova] [instance: 1954e745-783e-417e-aafc-265da55066cf] Refreshing instance network info cache due to event network-changed-4dba8130-7cfd-4940-86fb-303a6f32e278. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 798.181596] env[64020]: DEBUG oslo_concurrency.lockutils [req-2feb2b42-994d-46c0-b4bb-ecf481fbf7ad req-98c0c86d-925e-4f51-9e06-909b8d211634 service nova] Acquiring lock "refresh_cache-1954e745-783e-417e-aafc-265da55066cf" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 798.239321] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.496s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 798.239737] env[64020]: DEBUG nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 798.242107] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg ad95cf384c624862a8219dc684788678 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 798.243243] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.047s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 798.244910] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 27ba9f8043644080b2042be65858ec7a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 798.277527] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad95cf384c624862a8219dc684788678 [ 798.278201] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27ba9f8043644080b2042be65858ec7a [ 798.342188] env[64020]: DEBUG nova.network.neutron [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 798.360131] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Releasing lock "refresh_cache-b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 798.361154] env[64020]: DEBUG nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 798.361154] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e0394056-d4f0-47c9-8ccd-532e2296ea04 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.370376] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-954b7f92-7582-472f-97cd-c1910dbbf466 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.393456] env[64020]: WARNING nova.virt.vmwareapi.driver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Instance does not exists. Proceeding to delete instance properties on datastore: nova.exception.InstanceNotFound: Instance b89f34cd-bd7b-4f22-8e6f-a167d9f01af8 could not be found. [ 798.393674] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 798.393938] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e00a1a7a-4110-4988-8689-b27a4cd65cf3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.401225] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbe7290b-13fb-4d64-b0f6-c152246eeec9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.423694] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b89f34cd-bd7b-4f22-8e6f-a167d9f01af8 could not be found. [ 798.423945] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 798.424162] env[64020]: INFO nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Took 0.06 seconds to destroy the instance on the hypervisor. [ 798.424484] env[64020]: DEBUG oslo.service.loopingcall [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 798.424727] env[64020]: DEBUG nova.compute.manager [-] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 798.424824] env[64020]: DEBUG nova.network.neutron [-] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 798.428942] env[64020]: DEBUG nova.network.neutron [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 798.429568] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 1236ef89b21649d3b76ed6ad2da249c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 798.437930] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1236ef89b21649d3b76ed6ad2da249c1 [ 798.438776] env[64020]: DEBUG nova.network.neutron [-] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 798.439222] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a2db002eeceb4ec5b20e6ff8e8a9bf60 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 798.446597] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2db002eeceb4ec5b20e6ff8e8a9bf60 [ 798.747691] env[64020]: DEBUG nova.compute.utils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 798.748357] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg df266cc2dca64fc5922771f13f36a2cc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 798.752130] env[64020]: DEBUG nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 798.752294] env[64020]: DEBUG nova.network.neutron [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 798.758058] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg df266cc2dca64fc5922771f13f36a2cc [ 798.789954] env[64020]: DEBUG nova.policy [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd701c5273a9346a79001621429b93857', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2e4cb67e2820456ebab4033a4455a3c3', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 798.931308] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Releasing lock "refresh_cache-1954e745-783e-417e-aafc-265da55066cf" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 798.931722] env[64020]: DEBUG nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 798.931908] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 798.932889] env[64020]: DEBUG oslo_concurrency.lockutils [req-2feb2b42-994d-46c0-b4bb-ecf481fbf7ad req-98c0c86d-925e-4f51-9e06-909b8d211634 service nova] Acquired lock "refresh_cache-1954e745-783e-417e-aafc-265da55066cf" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 798.932889] env[64020]: DEBUG nova.network.neutron [req-2feb2b42-994d-46c0-b4bb-ecf481fbf7ad req-98c0c86d-925e-4f51-9e06-909b8d211634 service nova] [instance: 1954e745-783e-417e-aafc-265da55066cf] Refreshing network info cache for port 4dba8130-7cfd-4940-86fb-303a6f32e278 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 798.932889] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-2feb2b42-994d-46c0-b4bb-ecf481fbf7ad req-98c0c86d-925e-4f51-9e06-909b8d211634 service nova] Expecting reply to msg 22ceea040bc74172880ba3f11484374f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 798.933609] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-40ef1fbf-4a8b-4e90-b295-59959aa83409 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.941177] env[64020]: DEBUG nova.network.neutron [-] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 798.941566] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg cad522acb58e4056b9710ce351b50ab2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 798.942623] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 22ceea040bc74172880ba3f11484374f [ 798.947615] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f1fd91c-54b6-4eb0-84f5-3f91f7cad40b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.958359] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cad522acb58e4056b9710ce351b50ab2 [ 798.959172] env[64020]: INFO nova.compute.manager [-] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Took 0.53 seconds to deallocate network for instance. [ 798.974291] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1954e745-783e-417e-aafc-265da55066cf could not be found. [ 798.974603] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 798.974790] env[64020]: INFO nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Took 0.04 seconds to destroy the instance on the hypervisor. [ 798.975036] env[64020]: DEBUG oslo.service.loopingcall [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 798.977607] env[64020]: DEBUG nova.compute.manager [-] [instance: 1954e745-783e-417e-aafc-265da55066cf] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 798.978153] env[64020]: DEBUG nova.network.neutron [-] [instance: 1954e745-783e-417e-aafc-265da55066cf] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 798.997280] env[64020]: DEBUG nova.network.neutron [-] [instance: 1954e745-783e-417e-aafc-265da55066cf] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 798.997809] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 35d0ee064b65426bb13031db66acbb13 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 799.012065] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35d0ee064b65426bb13031db66acbb13 [ 799.094607] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e813c110-6fab-4232-b15c-3a41ad1bed59 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.103385] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93fd48a3-5a77-4a22-8bac-0bdc84d7532d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.135539] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95b9f958-a825-4dc9-97e0-bffdbc6db085 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.142546] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a3124d0-8d75-4421-a359-426924761fc2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.157088] env[64020]: DEBUG nova.compute.provider_tree [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 799.157570] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg b1285d8dc0e4451badd38a124a54e327 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 799.171466] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1285d8dc0e4451badd38a124a54e327 [ 799.175616] env[64020]: DEBUG nova.network.neutron [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Successfully created port: 97cd1637-d9ff-46b3-a34d-7ec120db0ffc {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 799.252877] env[64020]: DEBUG nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 799.254809] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 02ff7e7ad4d1489293c1442a2d4fef7b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 799.289509] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02ff7e7ad4d1489293c1442a2d4fef7b [ 799.463637] env[64020]: DEBUG nova.network.neutron [req-2feb2b42-994d-46c0-b4bb-ecf481fbf7ad req-98c0c86d-925e-4f51-9e06-909b8d211634 service nova] [instance: 1954e745-783e-417e-aafc-265da55066cf] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 799.502399] env[64020]: DEBUG nova.network.neutron [-] [instance: 1954e745-783e-417e-aafc-265da55066cf] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 799.502882] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 029b66030826463a8d23c023b0b4cadd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 799.511578] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 029b66030826463a8d23c023b0b4cadd [ 799.529558] env[64020]: INFO nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Took 0.57 seconds to detach 1 volumes for instance. [ 799.538367] env[64020]: DEBUG nova.compute.claims [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 799.538836] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 799.565122] env[64020]: DEBUG nova.network.neutron [req-2feb2b42-994d-46c0-b4bb-ecf481fbf7ad req-98c0c86d-925e-4f51-9e06-909b8d211634 service nova] [instance: 1954e745-783e-417e-aafc-265da55066cf] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 799.565738] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-2feb2b42-994d-46c0-b4bb-ecf481fbf7ad req-98c0c86d-925e-4f51-9e06-909b8d211634 service nova] Expecting reply to msg 97d261a8b51145f4a8303a6a4b90074b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 799.577734] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 97d261a8b51145f4a8303a6a4b90074b [ 799.669679] env[64020]: DEBUG nova.scheduler.client.report [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 799.669679] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 80f3bb3581fe4e2a8bd6698b150ca338 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 799.681219] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80f3bb3581fe4e2a8bd6698b150ca338 [ 799.759271] env[64020]: INFO nova.virt.block_device [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Booting with volume 20a1ae7e-f854-43e8-b969-679e6cd75aba at /dev/sda [ 799.796561] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9a611723-def8-43eb-9ce7-0a3818aeff5e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.803861] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31b28b31-3779-4381-bceb-4c1e130c188d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.827161] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3a8d5900-d6fa-4ec9-bf68-58b4ebb9d6e3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.840263] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09f98bcb-fd6e-4e1e-9858-934fef4a9ee5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.859640] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7cab1b8-8442-4533-a682-5bf79da131fe {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.865495] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf50e798-5717-42c1-87bc-da007a3cd02b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.877663] env[64020]: DEBUG nova.virt.block_device [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Updating existing volume attachment record: 48ddee46-69e8-4d0c-9fd4-b88d2652fc7d {{(pid=64020) _volume_attach /opt/stack/nova/nova/virt/block_device.py:665}} [ 800.011778] env[64020]: INFO nova.compute.manager [-] [instance: 1954e745-783e-417e-aafc-265da55066cf] Took 1.03 seconds to deallocate network for instance. [ 800.014185] env[64020]: DEBUG nova.compute.claims [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 800.014361] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.068241] env[64020]: DEBUG oslo_concurrency.lockutils [req-2feb2b42-994d-46c0-b4bb-ecf481fbf7ad req-98c0c86d-925e-4f51-9e06-909b8d211634 service nova] Releasing lock "refresh_cache-1954e745-783e-417e-aafc-265da55066cf" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 800.068521] env[64020]: DEBUG nova.compute.manager [req-2feb2b42-994d-46c0-b4bb-ecf481fbf7ad req-98c0c86d-925e-4f51-9e06-909b8d211634 service nova] [instance: 1954e745-783e-417e-aafc-265da55066cf] Received event network-vif-deleted-4dba8130-7cfd-4940-86fb-303a6f32e278 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 800.171482] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.927s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.171482] env[64020]: ERROR nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 46ff2091-6fa7-4aac-bfa0-2fdb8087185d, please check neutron logs for more information. [ 800.171482] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Traceback (most recent call last): [ 800.171482] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 800.171482] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] self.driver.spawn(context, instance, image_meta, [ 800.171482] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 800.171482] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] self._vmops.spawn(context, instance, image_meta, injected_files, [ 800.171482] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 800.171482] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] vm_ref = self.build_virtual_machine(instance, [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] vif_infos = vmwarevif.get_vif_info(self._session, [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] for vif in network_info: [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] return self._sync_wrapper(fn, *args, **kwargs) [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] self.wait() [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] self[:] = self._gt.wait() [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] return self._exit_event.wait() [ 800.171860] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] current.throw(*self._exc) [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] result = function(*args, **kwargs) [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] return func(*args, **kwargs) [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] raise e [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] nwinfo = self.network_api.allocate_for_instance( [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] created_port_ids = self._update_ports_for_instance( [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 800.172538] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] with excutils.save_and_reraise_exception(): [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] self.force_reraise() [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] raise self.value [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] updated_port = self._update_port( [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] _ensure_no_port_binding_failure(port) [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] raise exception.PortBindingFailed(port_id=port['id']) [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] nova.exception.PortBindingFailed: Binding failed for port 46ff2091-6fa7-4aac-bfa0-2fdb8087185d, please check neutron logs for more information. [ 800.172950] env[64020]: ERROR nova.compute.manager [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] [ 800.173349] env[64020]: DEBUG nova.compute.utils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Binding failed for port 46ff2091-6fa7-4aac-bfa0-2fdb8087185d, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 800.173450] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.276s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.174866] env[64020]: INFO nova.compute.claims [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 800.176390] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 8b51e3ca2b7d401caeb6695c1133b346 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 800.177512] env[64020]: DEBUG nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Build of instance 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa was re-scheduled: Binding failed for port 46ff2091-6fa7-4aac-bfa0-2fdb8087185d, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 800.177913] env[64020]: DEBUG nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 800.178129] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquiring lock "refresh_cache-2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 800.178273] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Acquired lock "refresh_cache-2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 800.178429] env[64020]: DEBUG nova.network.neutron [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 800.178771] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 71813d3563074121a566de97f73239c9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 800.185403] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 71813d3563074121a566de97f73239c9 [ 800.188144] env[64020]: DEBUG nova.compute.manager [req-35a18fdf-1a61-4fbf-b5e6-66c6bf80475e req-8fbd5d37-70e6-4eb3-95c9-6e6a4552c9b1 service nova] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Received event network-changed-97cd1637-d9ff-46b3-a34d-7ec120db0ffc {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 800.188396] env[64020]: DEBUG nova.compute.manager [req-35a18fdf-1a61-4fbf-b5e6-66c6bf80475e req-8fbd5d37-70e6-4eb3-95c9-6e6a4552c9b1 service nova] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Refreshing instance network info cache due to event network-changed-97cd1637-d9ff-46b3-a34d-7ec120db0ffc. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 800.188520] env[64020]: DEBUG oslo_concurrency.lockutils [req-35a18fdf-1a61-4fbf-b5e6-66c6bf80475e req-8fbd5d37-70e6-4eb3-95c9-6e6a4552c9b1 service nova] Acquiring lock "refresh_cache-5ccefb28-25a8-4876-a31c-58c7edfc9570" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 800.188659] env[64020]: DEBUG oslo_concurrency.lockutils [req-35a18fdf-1a61-4fbf-b5e6-66c6bf80475e req-8fbd5d37-70e6-4eb3-95c9-6e6a4552c9b1 service nova] Acquired lock "refresh_cache-5ccefb28-25a8-4876-a31c-58c7edfc9570" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 800.188853] env[64020]: DEBUG nova.network.neutron [req-35a18fdf-1a61-4fbf-b5e6-66c6bf80475e req-8fbd5d37-70e6-4eb3-95c9-6e6a4552c9b1 service nova] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Refreshing network info cache for port 97cd1637-d9ff-46b3-a34d-7ec120db0ffc {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 800.189169] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-35a18fdf-1a61-4fbf-b5e6-66c6bf80475e req-8fbd5d37-70e6-4eb3-95c9-6e6a4552c9b1 service nova] Expecting reply to msg a2f8a991ed9349e1ae42d65cd14c6fec in queue reply_57893177120949e6a93cb88e15cd42b4 [ 800.198768] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2f8a991ed9349e1ae42d65cd14c6fec [ 800.226558] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b51e3ca2b7d401caeb6695c1133b346 [ 800.298171] env[64020]: ERROR nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 97cd1637-d9ff-46b3-a34d-7ec120db0ffc, please check neutron logs for more information. [ 800.298171] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 800.298171] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 800.298171] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 800.298171] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 800.298171] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 800.298171] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 800.298171] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 800.298171] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 800.298171] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 800.298171] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 800.298171] env[64020]: ERROR nova.compute.manager raise self.value [ 800.298171] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 800.298171] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 800.298171] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 800.298171] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 800.298713] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 800.298713] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 800.298713] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 97cd1637-d9ff-46b3-a34d-7ec120db0ffc, please check neutron logs for more information. [ 800.298713] env[64020]: ERROR nova.compute.manager [ 800.298713] env[64020]: Traceback (most recent call last): [ 800.298713] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 800.298713] env[64020]: listener.cb(fileno) [ 800.298713] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 800.298713] env[64020]: result = function(*args, **kwargs) [ 800.298713] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 800.298713] env[64020]: return func(*args, **kwargs) [ 800.298713] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 800.298713] env[64020]: raise e [ 800.298713] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 800.298713] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 800.298713] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 800.298713] env[64020]: created_port_ids = self._update_ports_for_instance( [ 800.298713] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 800.298713] env[64020]: with excutils.save_and_reraise_exception(): [ 800.298713] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 800.298713] env[64020]: self.force_reraise() [ 800.298713] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 800.298713] env[64020]: raise self.value [ 800.298713] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 800.298713] env[64020]: updated_port = self._update_port( [ 800.298713] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 800.298713] env[64020]: _ensure_no_port_binding_failure(port) [ 800.298713] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 800.298713] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 800.299662] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 97cd1637-d9ff-46b3-a34d-7ec120db0ffc, please check neutron logs for more information. [ 800.299662] env[64020]: Removing descriptor: 16 [ 800.427477] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 231523be07c14d3782d75617e3c18441 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 800.437161] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 231523be07c14d3782d75617e3c18441 [ 800.681699] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg ae26d15db3b94ac5b821b20a30144bd6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 800.690182] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae26d15db3b94ac5b821b20a30144bd6 [ 800.704583] env[64020]: DEBUG nova.network.neutron [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 800.706985] env[64020]: DEBUG nova.network.neutron [req-35a18fdf-1a61-4fbf-b5e6-66c6bf80475e req-8fbd5d37-70e6-4eb3-95c9-6e6a4552c9b1 service nova] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 800.785758] env[64020]: DEBUG nova.network.neutron [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 800.786308] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 0864e9a242074e6b8f58826c87d3416f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 800.794237] env[64020]: DEBUG nova.network.neutron [req-35a18fdf-1a61-4fbf-b5e6-66c6bf80475e req-8fbd5d37-70e6-4eb3-95c9-6e6a4552c9b1 service nova] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 800.794690] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-35a18fdf-1a61-4fbf-b5e6-66c6bf80475e req-8fbd5d37-70e6-4eb3-95c9-6e6a4552c9b1 service nova] Expecting reply to msg 85180954202340079dc35b7c81b9f01e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 800.795629] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0864e9a242074e6b8f58826c87d3416f [ 800.802367] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 85180954202340079dc35b7c81b9f01e [ 800.980712] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 86a3fde7d7944ee5b33e24974219cb95 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 800.990497] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86a3fde7d7944ee5b33e24974219cb95 [ 801.288043] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Releasing lock "refresh_cache-2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 801.288281] env[64020]: DEBUG nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 801.288447] env[64020]: DEBUG nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 801.288621] env[64020]: DEBUG nova.network.neutron [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 801.297300] env[64020]: DEBUG oslo_concurrency.lockutils [req-35a18fdf-1a61-4fbf-b5e6-66c6bf80475e req-8fbd5d37-70e6-4eb3-95c9-6e6a4552c9b1 service nova] Releasing lock "refresh_cache-5ccefb28-25a8-4876-a31c-58c7edfc9570" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 801.305761] env[64020]: DEBUG nova.network.neutron [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 801.306311] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 58cf0ee3bdb14cf4b99770d6e3efa2e4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 801.314786] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 58cf0ee3bdb14cf4b99770d6e3efa2e4 [ 801.485131] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 5a5072c6f8f6439c94f8fa265da0818f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 801.520145] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a5072c6f8f6439c94f8fa265da0818f [ 801.529907] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c6a4afc-899a-49c7-8cb7-2e2602e27dd2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.538812] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-941f4e1c-5387-4981-a816-e892e95766c1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.568527] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-63198c8e-4c4e-413a-a11c-ae4f3c62dbc1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.575705] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28451c59-cd91-4828-95b2-30d04a7e2652 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.590489] env[64020]: DEBUG nova.compute.provider_tree [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 801.590975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 986379a4db3a46b196dd8a98fc48da91 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 801.598717] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 986379a4db3a46b196dd8a98fc48da91 [ 801.810340] env[64020]: DEBUG nova.network.neutron [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 801.811197] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 2ea5eca056f04457a0b73b94536812a0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 801.819546] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ea5eca056f04457a0b73b94536812a0 [ 801.988257] env[64020]: DEBUG nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 801.988781] env[64020]: DEBUG nova.virt.hardware [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 801.988987] env[64020]: DEBUG nova.virt.hardware [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 801.989274] env[64020]: DEBUG nova.virt.hardware [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 801.989540] env[64020]: DEBUG nova.virt.hardware [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 801.989697] env[64020]: DEBUG nova.virt.hardware [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 801.989842] env[64020]: DEBUG nova.virt.hardware [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 801.990048] env[64020]: DEBUG nova.virt.hardware [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 801.990203] env[64020]: DEBUG nova.virt.hardware [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 801.990366] env[64020]: DEBUG nova.virt.hardware [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 801.990523] env[64020]: DEBUG nova.virt.hardware [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 801.990761] env[64020]: DEBUG nova.virt.hardware [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 801.991612] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06b1d63e-f43a-409a-a49a-63de5308d70a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.999602] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f866b7e4-3a29-4466-a8d1-eea7e51ef042 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.012742] env[64020]: ERROR nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 97cd1637-d9ff-46b3-a34d-7ec120db0ffc, please check neutron logs for more information. [ 802.012742] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Traceback (most recent call last): [ 802.012742] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 802.012742] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] yield resources [ 802.012742] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 802.012742] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] self.driver.spawn(context, instance, image_meta, [ 802.012742] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 802.012742] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] self._vmops.spawn(context, instance, image_meta, injected_files, [ 802.012742] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 802.012742] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] vm_ref = self.build_virtual_machine(instance, [ 802.012742] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] vif_infos = vmwarevif.get_vif_info(self._session, [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] for vif in network_info: [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] return self._sync_wrapper(fn, *args, **kwargs) [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] self.wait() [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] self[:] = self._gt.wait() [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] return self._exit_event.wait() [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 802.013185] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] current.throw(*self._exc) [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] result = function(*args, **kwargs) [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] return func(*args, **kwargs) [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] raise e [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] nwinfo = self.network_api.allocate_for_instance( [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] created_port_ids = self._update_ports_for_instance( [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] with excutils.save_and_reraise_exception(): [ 802.013594] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 802.014024] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] self.force_reraise() [ 802.014024] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 802.014024] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] raise self.value [ 802.014024] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 802.014024] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] updated_port = self._update_port( [ 802.014024] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 802.014024] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] _ensure_no_port_binding_failure(port) [ 802.014024] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 802.014024] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] raise exception.PortBindingFailed(port_id=port['id']) [ 802.014024] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] nova.exception.PortBindingFailed: Binding failed for port 97cd1637-d9ff-46b3-a34d-7ec120db0ffc, please check neutron logs for more information. [ 802.014024] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] [ 802.014024] env[64020]: INFO nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Terminating instance [ 802.015109] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Acquiring lock "refresh_cache-5ccefb28-25a8-4876-a31c-58c7edfc9570" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 802.015363] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Acquired lock "refresh_cache-5ccefb28-25a8-4876-a31c-58c7edfc9570" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 802.015550] env[64020]: DEBUG nova.network.neutron [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 802.016164] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg bde584d841f34982a5605f3f0d5aae45 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 802.022625] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bde584d841f34982a5605f3f0d5aae45 [ 802.093806] env[64020]: DEBUG nova.scheduler.client.report [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 802.096193] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg f81370ef38734b2fbb58b45b4832c60f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 802.112875] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f81370ef38734b2fbb58b45b4832c60f [ 802.228937] env[64020]: DEBUG nova.compute.manager [req-6ab125ab-c3a7-4fc2-90a6-a5d3680b738f req-95e24d6d-aa8e-4336-ae9c-0d0dc5f3eb65 service nova] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Received event network-vif-deleted-97cd1637-d9ff-46b3-a34d-7ec120db0ffc {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 802.313549] env[64020]: INFO nova.compute.manager [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] [instance: 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa] Took 1.02 seconds to deallocate network for instance. [ 802.315294] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg c2b0050ba5894fecb78535d10b60b499 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 802.345907] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2b0050ba5894fecb78535d10b60b499 [ 802.531342] env[64020]: DEBUG nova.network.neutron [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 802.598768] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.425s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.599267] env[64020]: DEBUG nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 802.601016] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 6f34eb8280014c538d83fbd9cf8cac49 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 802.602804] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.734s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.604175] env[64020]: INFO nova.compute.claims [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 802.605642] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg c4eb357f064b4556bd7eb10ab6260bc4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 802.610006] env[64020]: DEBUG nova.network.neutron [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 802.610399] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 05b11aab21e544328a349f520330dd4d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 802.620405] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05b11aab21e544328a349f520330dd4d [ 802.636802] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f34eb8280014c538d83fbd9cf8cac49 [ 802.649276] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c4eb357f064b4556bd7eb10ab6260bc4 [ 802.820617] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 835bb1234d3445268de2d1e4855f9153 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 802.851577] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 835bb1234d3445268de2d1e4855f9153 [ 803.104662] env[64020]: DEBUG nova.compute.utils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 803.105294] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg af426dbb148e42239eabe7ae4ad9c9d6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 803.106190] env[64020]: DEBUG nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 803.107353] env[64020]: DEBUG nova.network.neutron [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 803.114417] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 5a07b79bbf544293acfda580b2a784bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 803.115291] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Releasing lock "refresh_cache-5ccefb28-25a8-4876-a31c-58c7edfc9570" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 803.116040] env[64020]: DEBUG nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 803.116398] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-20cdf411-9be0-4ee9-8aa5-90239f386cf7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.119349] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af426dbb148e42239eabe7ae4ad9c9d6 [ 803.122191] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a07b79bbf544293acfda580b2a784bc [ 803.129748] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8277e4a-0f03-4d15-8b75-548dda640eff {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.151449] env[64020]: WARNING nova.virt.vmwareapi.driver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Instance does not exists. Proceeding to delete instance properties on datastore: nova.exception.InstanceNotFound: Instance 5ccefb28-25a8-4876-a31c-58c7edfc9570 could not be found. [ 803.151707] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 803.152009] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4950af3d-54fa-421c-bbb7-890d8160e4fa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.159419] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03542891-1a74-4349-bfd9-999d3e98bf46 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.170305] env[64020]: DEBUG nova.policy [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8ed3a2184b5b453ba8a0e2b304337406', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '1d7711904b504291af77b20d8abde544', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 803.182234] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5ccefb28-25a8-4876-a31c-58c7edfc9570 could not be found. [ 803.182491] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 803.182709] env[64020]: INFO nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Took 0.07 seconds to destroy the instance on the hypervisor. [ 803.182988] env[64020]: DEBUG oslo.service.loopingcall [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 803.183237] env[64020]: DEBUG nova.compute.manager [-] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 803.183355] env[64020]: DEBUG nova.network.neutron [-] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 803.197013] env[64020]: DEBUG nova.network.neutron [-] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 803.197483] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ff667d6e9d5d4a649d76836da541505f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 803.205089] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff667d6e9d5d4a649d76836da541505f [ 803.344180] env[64020]: INFO nova.scheduler.client.report [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Deleted allocations for instance 2e8d4bb1-9a84-4d1e-9df3-5169d4066daa [ 803.349849] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Expecting reply to msg 40e7e08dc46f42809ef56488a7908cba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 803.371661] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40e7e08dc46f42809ef56488a7908cba [ 803.494867] env[64020]: DEBUG nova.network.neutron [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Successfully created port: e13ceee3-08b5-4500-b589-817077fa9ddb {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 803.610407] env[64020]: DEBUG nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 803.612284] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 4b812f819e1342f78c4672aa83799d24 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 803.680666] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4b812f819e1342f78c4672aa83799d24 [ 803.700383] env[64020]: DEBUG nova.network.neutron [-] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 803.700728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 4f2fe44a21c14479b36a1a983a975994 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 803.714169] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f2fe44a21c14479b36a1a983a975994 [ 803.854356] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a39a12d0-a462-4a3f-90dd-2b02dd502bf5 tempest-DeleteServersAdminTestJSON-1407500400 tempest-DeleteServersAdminTestJSON-1407500400-project-member] Lock "2e8d4bb1-9a84-4d1e-9df3-5169d4066daa" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 165.415s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 803.854887] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg af072420b6c249299e505b81c8ffe5b5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 803.865121] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af072420b6c249299e505b81c8ffe5b5 [ 803.923722] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2eb8738-3e45-4372-a5fa-d0d53ef37333 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.931618] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8c33938-2410-4a23-b237-5ee95f6bd650 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.965792] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2adaa3ea-f74b-4b43-a2bd-2a47446fe187 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.975932] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b729b97c-ca48-4ff2-bfcf-851b366215a7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.990302] env[64020]: DEBUG nova.compute.provider_tree [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 803.990818] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg c520df281bb44546b60ddd3a86a335a3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 803.997656] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c520df281bb44546b60ddd3a86a335a3 [ 804.118382] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg bf604f1fe39946189e9a14bb7a809b8b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 804.147985] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf604f1fe39946189e9a14bb7a809b8b [ 804.169159] env[64020]: DEBUG nova.compute.manager [req-716829d2-623b-475d-bcdf-c4dc071dd438 req-79d42b10-9fbc-4b7c-9c77-009d9eb3afa9 service nova] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Received event network-changed-e13ceee3-08b5-4500-b589-817077fa9ddb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 804.169337] env[64020]: DEBUG nova.compute.manager [req-716829d2-623b-475d-bcdf-c4dc071dd438 req-79d42b10-9fbc-4b7c-9c77-009d9eb3afa9 service nova] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Refreshing instance network info cache due to event network-changed-e13ceee3-08b5-4500-b589-817077fa9ddb. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 804.169543] env[64020]: DEBUG oslo_concurrency.lockutils [req-716829d2-623b-475d-bcdf-c4dc071dd438 req-79d42b10-9fbc-4b7c-9c77-009d9eb3afa9 service nova] Acquiring lock "refresh_cache-a39b40c8-0340-4778-8b43-f29ed51664bc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 804.169675] env[64020]: DEBUG oslo_concurrency.lockutils [req-716829d2-623b-475d-bcdf-c4dc071dd438 req-79d42b10-9fbc-4b7c-9c77-009d9eb3afa9 service nova] Acquired lock "refresh_cache-a39b40c8-0340-4778-8b43-f29ed51664bc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 804.169826] env[64020]: DEBUG nova.network.neutron [req-716829d2-623b-475d-bcdf-c4dc071dd438 req-79d42b10-9fbc-4b7c-9c77-009d9eb3afa9 service nova] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Refreshing network info cache for port e13ceee3-08b5-4500-b589-817077fa9ddb {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 804.170249] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-716829d2-623b-475d-bcdf-c4dc071dd438 req-79d42b10-9fbc-4b7c-9c77-009d9eb3afa9 service nova] Expecting reply to msg 910542f2e7c3485f942699c25943728e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 804.178171] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 910542f2e7c3485f942699c25943728e [ 804.205908] env[64020]: INFO nova.compute.manager [-] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Took 1.02 seconds to deallocate network for instance. [ 804.361397] env[64020]: DEBUG nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 804.361397] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 0a49367bea7149498ffeaaf464c52a7e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 804.382093] env[64020]: ERROR nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e13ceee3-08b5-4500-b589-817077fa9ddb, please check neutron logs for more information. [ 804.382093] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 804.382093] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 804.382093] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 804.382093] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 804.382093] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 804.382093] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 804.382093] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 804.382093] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 804.382093] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 804.382093] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 804.382093] env[64020]: ERROR nova.compute.manager raise self.value [ 804.382093] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 804.382093] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 804.382093] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 804.382093] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 804.382647] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 804.382647] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 804.382647] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e13ceee3-08b5-4500-b589-817077fa9ddb, please check neutron logs for more information. [ 804.382647] env[64020]: ERROR nova.compute.manager [ 804.382647] env[64020]: Traceback (most recent call last): [ 804.382647] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 804.382647] env[64020]: listener.cb(fileno) [ 804.382647] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 804.382647] env[64020]: result = function(*args, **kwargs) [ 804.382647] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 804.382647] env[64020]: return func(*args, **kwargs) [ 804.382647] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 804.382647] env[64020]: raise e [ 804.382647] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 804.382647] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 804.382647] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 804.382647] env[64020]: created_port_ids = self._update_ports_for_instance( [ 804.382647] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 804.382647] env[64020]: with excutils.save_and_reraise_exception(): [ 804.382647] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 804.382647] env[64020]: self.force_reraise() [ 804.382647] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 804.382647] env[64020]: raise self.value [ 804.382647] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 804.382647] env[64020]: updated_port = self._update_port( [ 804.382647] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 804.382647] env[64020]: _ensure_no_port_binding_failure(port) [ 804.382647] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 804.382647] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 804.383510] env[64020]: nova.exception.PortBindingFailed: Binding failed for port e13ceee3-08b5-4500-b589-817077fa9ddb, please check neutron logs for more information. [ 804.383510] env[64020]: Removing descriptor: 18 [ 804.397017] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a49367bea7149498ffeaaf464c52a7e [ 804.493599] env[64020]: DEBUG nova.scheduler.client.report [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 804.496085] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 19fdd75baf6b4aa18fb0a9ee4fc08500 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 804.507251] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19fdd75baf6b4aa18fb0a9ee4fc08500 [ 804.623612] env[64020]: DEBUG nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 804.647216] env[64020]: DEBUG nova.virt.hardware [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 804.647459] env[64020]: DEBUG nova.virt.hardware [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 804.647611] env[64020]: DEBUG nova.virt.hardware [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 804.647787] env[64020]: DEBUG nova.virt.hardware [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 804.647926] env[64020]: DEBUG nova.virt.hardware [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 804.648081] env[64020]: DEBUG nova.virt.hardware [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 804.648295] env[64020]: DEBUG nova.virt.hardware [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 804.648451] env[64020]: DEBUG nova.virt.hardware [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 804.648612] env[64020]: DEBUG nova.virt.hardware [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 804.648773] env[64020]: DEBUG nova.virt.hardware [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 804.648948] env[64020]: DEBUG nova.virt.hardware [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 804.649828] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30ab20e1-a3bc-4baf-baa3-db84686e758e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 804.657946] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d167990-d642-49f2-9d0f-c4636ee37937 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 804.671111] env[64020]: ERROR nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e13ceee3-08b5-4500-b589-817077fa9ddb, please check neutron logs for more information. [ 804.671111] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Traceback (most recent call last): [ 804.671111] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 804.671111] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] yield resources [ 804.671111] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 804.671111] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] self.driver.spawn(context, instance, image_meta, [ 804.671111] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 804.671111] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 804.671111] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 804.671111] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] vm_ref = self.build_virtual_machine(instance, [ 804.671111] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] vif_infos = vmwarevif.get_vif_info(self._session, [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] for vif in network_info: [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] return self._sync_wrapper(fn, *args, **kwargs) [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] self.wait() [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] self[:] = self._gt.wait() [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] return self._exit_event.wait() [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 804.671514] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] current.throw(*self._exc) [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] result = function(*args, **kwargs) [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] return func(*args, **kwargs) [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] raise e [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] nwinfo = self.network_api.allocate_for_instance( [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] created_port_ids = self._update_ports_for_instance( [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] with excutils.save_and_reraise_exception(): [ 804.671902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 804.672465] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] self.force_reraise() [ 804.672465] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 804.672465] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] raise self.value [ 804.672465] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 804.672465] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] updated_port = self._update_port( [ 804.672465] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 804.672465] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] _ensure_no_port_binding_failure(port) [ 804.672465] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 804.672465] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] raise exception.PortBindingFailed(port_id=port['id']) [ 804.672465] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] nova.exception.PortBindingFailed: Binding failed for port e13ceee3-08b5-4500-b589-817077fa9ddb, please check neutron logs for more information. [ 804.672465] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] [ 804.672465] env[64020]: INFO nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Terminating instance [ 804.673412] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Acquiring lock "refresh_cache-a39b40c8-0340-4778-8b43-f29ed51664bc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 804.688873] env[64020]: DEBUG nova.network.neutron [req-716829d2-623b-475d-bcdf-c4dc071dd438 req-79d42b10-9fbc-4b7c-9c77-009d9eb3afa9 service nova] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 804.740863] env[64020]: DEBUG nova.network.neutron [req-716829d2-623b-475d-bcdf-c4dc071dd438 req-79d42b10-9fbc-4b7c-9c77-009d9eb3afa9 service nova] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 804.741092] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-716829d2-623b-475d-bcdf-c4dc071dd438 req-79d42b10-9fbc-4b7c-9c77-009d9eb3afa9 service nova] Expecting reply to msg 67eebb4fe4b4461da46c9e7ebf07bdc5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 804.750133] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 67eebb4fe4b4461da46c9e7ebf07bdc5 [ 804.762781] env[64020]: INFO nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Took 0.56 seconds to detach 1 volumes for instance. [ 804.764889] env[64020]: DEBUG nova.compute.claims [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 804.765055] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 804.879924] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 804.998718] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.396s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 804.999297] env[64020]: DEBUG nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 805.001025] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 7a3d55a322d24e01b632e5465f2f327b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 805.002105] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.228s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 805.007264] env[64020]: INFO nova.compute.claims [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 805.008343] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg c52a91d90d4e46b1b4574b4c6062f7e1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 805.034078] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a3d55a322d24e01b632e5465f2f327b [ 805.044515] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c52a91d90d4e46b1b4574b4c6062f7e1 [ 805.243205] env[64020]: DEBUG oslo_concurrency.lockutils [req-716829d2-623b-475d-bcdf-c4dc071dd438 req-79d42b10-9fbc-4b7c-9c77-009d9eb3afa9 service nova] Releasing lock "refresh_cache-a39b40c8-0340-4778-8b43-f29ed51664bc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 805.243697] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Acquired lock "refresh_cache-a39b40c8-0340-4778-8b43-f29ed51664bc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 805.243879] env[64020]: DEBUG nova.network.neutron [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 805.244361] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg ebde54fad1114888bd1e32dde49bacb3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 805.251455] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ebde54fad1114888bd1e32dde49bacb3 [ 805.512777] env[64020]: DEBUG nova.compute.utils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 805.512976] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 89da9bfeb6f24b76b4e9a46500d54e18 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 805.514586] env[64020]: DEBUG nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 805.515434] env[64020]: DEBUG nova.network.neutron [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 805.517827] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg ca548eabbba44d4e8fcc2acce4ef18ab in queue reply_57893177120949e6a93cb88e15cd42b4 [ 805.525301] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89da9bfeb6f24b76b4e9a46500d54e18 [ 805.525854] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca548eabbba44d4e8fcc2acce4ef18ab [ 805.569625] env[64020]: DEBUG nova.policy [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'bb072c99fa744420bcb9872abc7c4002', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f5cd2aad224046f8b63018de74ce7564', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 805.762038] env[64020]: DEBUG nova.network.neutron [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 805.816267] env[64020]: DEBUG nova.network.neutron [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Successfully created port: bcd04248-654c-44de-ab87-7f1272bc06d3 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 805.847454] env[64020]: DEBUG nova.network.neutron [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 805.847836] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg f54af9e26256444b8a0287826af08671 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 805.856484] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f54af9e26256444b8a0287826af08671 [ 806.020397] env[64020]: DEBUG nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 806.023477] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 7f1a46455f9b4817b08fc77c25f8e5dc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 806.059738] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f1a46455f9b4817b08fc77c25f8e5dc [ 806.349775] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Releasing lock "refresh_cache-a39b40c8-0340-4778-8b43-f29ed51664bc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 806.350250] env[64020]: DEBUG nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 806.350609] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 806.353638] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-46f273da-6bdb-4920-b6f7-d156129002b9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.363395] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a231e467-5135-4e10-9f3e-31c09315aff4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.394841] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a39b40c8-0340-4778-8b43-f29ed51664bc could not be found. [ 806.395083] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 806.395265] env[64020]: INFO nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Took 0.04 seconds to destroy the instance on the hypervisor. [ 806.395512] env[64020]: DEBUG oslo.service.loopingcall [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 806.395725] env[64020]: DEBUG nova.compute.manager [-] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 806.395818] env[64020]: DEBUG nova.network.neutron [-] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 806.414074] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a8c476f-4a82-4fa4-912a-a481bdd293b3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.422024] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8c6c1cc-93c3-433a-9385-d22a41ee8cba {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.427776] env[64020]: DEBUG nova.network.neutron [-] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 806.428321] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 2bb455764e554671adbef03ce874d600 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 806.429913] env[64020]: DEBUG nova.compute.manager [req-66b04ca9-9bb5-47e3-80de-fd589bebdb62 req-082b77b1-2acd-464b-954f-74eeb029b9f4 service nova] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Received event network-vif-deleted-e13ceee3-08b5-4500-b589-817077fa9ddb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 806.463421] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2bb455764e554671adbef03ce874d600 [ 806.464289] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42571872-21f8-4dcf-95f1-7484ba500797 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.473398] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22fbec9e-538e-411a-a3a8-a1a7f5f49492 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.489038] env[64020]: DEBUG nova.compute.provider_tree [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 806.489724] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 7865a925e84843118dfd25064c801c80 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 806.500903] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7865a925e84843118dfd25064c801c80 [ 806.530209] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg ed7be69c9cc54fedbdbb1dde925f6cf3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 806.565202] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed7be69c9cc54fedbdbb1dde925f6cf3 [ 806.772734] env[64020]: ERROR nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port bcd04248-654c-44de-ab87-7f1272bc06d3, please check neutron logs for more information. [ 806.772734] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 806.772734] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 806.772734] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 806.772734] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 806.772734] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 806.772734] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 806.772734] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 806.772734] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 806.772734] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 806.772734] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 806.772734] env[64020]: ERROR nova.compute.manager raise self.value [ 806.772734] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 806.772734] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 806.772734] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 806.772734] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 806.773201] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 806.773201] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 806.773201] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port bcd04248-654c-44de-ab87-7f1272bc06d3, please check neutron logs for more information. [ 806.773201] env[64020]: ERROR nova.compute.manager [ 806.773201] env[64020]: Traceback (most recent call last): [ 806.773201] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 806.773201] env[64020]: listener.cb(fileno) [ 806.773201] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 806.773201] env[64020]: result = function(*args, **kwargs) [ 806.773201] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 806.773201] env[64020]: return func(*args, **kwargs) [ 806.773201] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 806.773201] env[64020]: raise e [ 806.773201] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 806.773201] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 806.773201] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 806.773201] env[64020]: created_port_ids = self._update_ports_for_instance( [ 806.773201] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 806.773201] env[64020]: with excutils.save_and_reraise_exception(): [ 806.773201] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 806.773201] env[64020]: self.force_reraise() [ 806.773201] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 806.773201] env[64020]: raise self.value [ 806.773201] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 806.773201] env[64020]: updated_port = self._update_port( [ 806.773201] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 806.773201] env[64020]: _ensure_no_port_binding_failure(port) [ 806.773201] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 806.773201] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 806.774003] env[64020]: nova.exception.PortBindingFailed: Binding failed for port bcd04248-654c-44de-ab87-7f1272bc06d3, please check neutron logs for more information. [ 806.774003] env[64020]: Removing descriptor: 18 [ 806.931499] env[64020]: DEBUG nova.network.neutron [-] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 806.931977] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7f85dfe87bd84a0faa9d745f1e5c246e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 806.941513] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f85dfe87bd84a0faa9d745f1e5c246e [ 806.993748] env[64020]: DEBUG nova.scheduler.client.report [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 806.996225] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 0d1037ea9a0d40b296de92e2caf8d95a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 807.020490] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0d1037ea9a0d40b296de92e2caf8d95a [ 807.032797] env[64020]: DEBUG nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 807.085537] env[64020]: DEBUG nova.virt.hardware [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 807.085537] env[64020]: DEBUG nova.virt.hardware [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 807.085537] env[64020]: DEBUG nova.virt.hardware [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 807.085787] env[64020]: DEBUG nova.virt.hardware [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 807.085787] env[64020]: DEBUG nova.virt.hardware [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 807.085787] env[64020]: DEBUG nova.virt.hardware [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 807.085787] env[64020]: DEBUG nova.virt.hardware [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 807.085787] env[64020]: DEBUG nova.virt.hardware [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 807.086083] env[64020]: DEBUG nova.virt.hardware [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 807.086083] env[64020]: DEBUG nova.virt.hardware [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 807.086083] env[64020]: DEBUG nova.virt.hardware [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 807.089763] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fdb9f721-7326-4a7b-b4d6-22bf2a35b5b2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 807.095002] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5889317-ae53-4539-9d26-9522d6e94f26 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 807.109611] env[64020]: ERROR nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port bcd04248-654c-44de-ab87-7f1272bc06d3, please check neutron logs for more information. [ 807.109611] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Traceback (most recent call last): [ 807.109611] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 807.109611] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] yield resources [ 807.109611] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 807.109611] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] self.driver.spawn(context, instance, image_meta, [ 807.109611] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 807.109611] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 807.109611] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 807.109611] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] vm_ref = self.build_virtual_machine(instance, [ 807.109611] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] vif_infos = vmwarevif.get_vif_info(self._session, [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] for vif in network_info: [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] return self._sync_wrapper(fn, *args, **kwargs) [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] self.wait() [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] self[:] = self._gt.wait() [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] return self._exit_event.wait() [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 807.110169] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] current.throw(*self._exc) [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] result = function(*args, **kwargs) [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] return func(*args, **kwargs) [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] raise e [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] nwinfo = self.network_api.allocate_for_instance( [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] created_port_ids = self._update_ports_for_instance( [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] with excutils.save_and_reraise_exception(): [ 807.110593] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 807.110978] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] self.force_reraise() [ 807.110978] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 807.110978] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] raise self.value [ 807.110978] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 807.110978] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] updated_port = self._update_port( [ 807.110978] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 807.110978] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] _ensure_no_port_binding_failure(port) [ 807.110978] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 807.110978] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] raise exception.PortBindingFailed(port_id=port['id']) [ 807.110978] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] nova.exception.PortBindingFailed: Binding failed for port bcd04248-654c-44de-ab87-7f1272bc06d3, please check neutron logs for more information. [ 807.110978] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] [ 807.111686] env[64020]: INFO nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Terminating instance [ 807.114260] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Acquiring lock "refresh_cache-73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 807.114549] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Acquired lock "refresh_cache-73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 807.114826] env[64020]: DEBUG nova.network.neutron [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 807.115342] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 6da0b04333704e7787a576f98b12799a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 807.125304] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6da0b04333704e7787a576f98b12799a [ 807.434312] env[64020]: INFO nova.compute.manager [-] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Took 1.04 seconds to deallocate network for instance. [ 807.436585] env[64020]: DEBUG nova.compute.claims [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 807.436756] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 807.498276] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.496s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 807.498796] env[64020]: DEBUG nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 807.501005] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 4526ee4265e340f88d59b9fea6592fe8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 807.501976] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.802s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 807.512676] env[64020]: INFO nova.compute.claims [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 807.512676] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg d496af59f40a4337b0d7d5e01fbe05b2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 807.536637] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4526ee4265e340f88d59b9fea6592fe8 [ 807.545894] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d496af59f40a4337b0d7d5e01fbe05b2 [ 807.635083] env[64020]: DEBUG nova.network.neutron [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 807.730801] env[64020]: DEBUG nova.network.neutron [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 807.731398] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg fca1f3d0fb8740789a71c076f5fd7383 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 807.739874] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fca1f3d0fb8740789a71c076f5fd7383 [ 808.007805] env[64020]: DEBUG nova.compute.utils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 808.008482] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 8105922c101a42f3a465fcc599349ea4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 808.010495] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 0575fa58f88347108dfd6fb6639d2bbd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 808.011450] env[64020]: DEBUG nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 808.011613] env[64020]: DEBUG nova.network.neutron [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 808.023490] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8105922c101a42f3a465fcc599349ea4 [ 808.024413] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0575fa58f88347108dfd6fb6639d2bbd [ 808.069217] env[64020]: DEBUG nova.policy [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9c183cac5f44461daf639a0122e8fee2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6c97516aa6204d9dae64a28689a4d42c', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 808.234996] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Releasing lock "refresh_cache-73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 808.234996] env[64020]: DEBUG nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 808.234996] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 808.234996] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-6192d528-6df7-486c-a49f-80aa1343a9d7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.244071] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-288d2876-537d-4160-af69-7d3ba31d33dc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.265074] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c could not be found. [ 808.265275] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 808.265453] env[64020]: INFO nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Took 0.03 seconds to destroy the instance on the hypervisor. [ 808.265686] env[64020]: DEBUG oslo.service.loopingcall [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 808.265888] env[64020]: DEBUG nova.compute.manager [-] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 808.265981] env[64020]: DEBUG nova.network.neutron [-] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 808.284110] env[64020]: DEBUG nova.network.neutron [-] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 808.284659] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7027c0ce4fcf44e6b1cb404161ce622f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 808.297727] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7027c0ce4fcf44e6b1cb404161ce622f [ 808.439328] env[64020]: DEBUG nova.network.neutron [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Successfully created port: e83e0437-9379-4de3-b9fd-44e3086389c7 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 808.498655] env[64020]: DEBUG nova.compute.manager [req-df5ec41f-414e-45e1-9e13-c076cf511a06 req-dacfdbf3-e955-4dee-8134-cb0c5088381c service nova] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Received event network-changed-bcd04248-654c-44de-ab87-7f1272bc06d3 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 808.498655] env[64020]: DEBUG nova.compute.manager [req-df5ec41f-414e-45e1-9e13-c076cf511a06 req-dacfdbf3-e955-4dee-8134-cb0c5088381c service nova] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Refreshing instance network info cache due to event network-changed-bcd04248-654c-44de-ab87-7f1272bc06d3. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 808.498655] env[64020]: DEBUG oslo_concurrency.lockutils [req-df5ec41f-414e-45e1-9e13-c076cf511a06 req-dacfdbf3-e955-4dee-8134-cb0c5088381c service nova] Acquiring lock "refresh_cache-73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 808.498655] env[64020]: DEBUG oslo_concurrency.lockutils [req-df5ec41f-414e-45e1-9e13-c076cf511a06 req-dacfdbf3-e955-4dee-8134-cb0c5088381c service nova] Acquired lock "refresh_cache-73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 808.498655] env[64020]: DEBUG nova.network.neutron [req-df5ec41f-414e-45e1-9e13-c076cf511a06 req-dacfdbf3-e955-4dee-8134-cb0c5088381c service nova] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Refreshing network info cache for port bcd04248-654c-44de-ab87-7f1272bc06d3 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 808.498841] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-df5ec41f-414e-45e1-9e13-c076cf511a06 req-dacfdbf3-e955-4dee-8134-cb0c5088381c service nova] Expecting reply to msg 8cc16459e14143428a63794fd4e7aa9a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 808.505269] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8cc16459e14143428a63794fd4e7aa9a [ 808.512396] env[64020]: DEBUG nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 808.514827] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 896255610e924b5ba8eed49f148b4416 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 808.554849] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 896255610e924b5ba8eed49f148b4416 [ 808.794582] env[64020]: DEBUG nova.network.neutron [-] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 808.795034] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 909fdeb985bc4fa3925cedc990fe1fa7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 808.803030] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 909fdeb985bc4fa3925cedc990fe1fa7 [ 808.893366] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca58213c-e083-4b9d-bc6e-1e8b7f38dabf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.901714] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ee10ba2-d41a-4a5e-b463-2465dbb55272 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.944183] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c464f0df-4817-4b28-9018-b50c00964022 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.952713] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3666160-e11b-4619-b0e4-1011c2e6bce3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.968849] env[64020]: DEBUG nova.compute.provider_tree [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 808.969381] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 484457e85cf3480397e1e02ec3b526cf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 808.979951] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 484457e85cf3480397e1e02ec3b526cf [ 809.027923] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 03b6ef26cf64455a8f6e5e35108b9691 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 809.030936] env[64020]: DEBUG nova.network.neutron [req-df5ec41f-414e-45e1-9e13-c076cf511a06 req-dacfdbf3-e955-4dee-8134-cb0c5088381c service nova] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 809.064640] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03b6ef26cf64455a8f6e5e35108b9691 [ 809.215589] env[64020]: DEBUG nova.network.neutron [req-df5ec41f-414e-45e1-9e13-c076cf511a06 req-dacfdbf3-e955-4dee-8134-cb0c5088381c service nova] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 809.216127] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-df5ec41f-414e-45e1-9e13-c076cf511a06 req-dacfdbf3-e955-4dee-8134-cb0c5088381c service nova] Expecting reply to msg c38868d3ae8248e1ac650b2b4ed92e8f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 809.224453] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c38868d3ae8248e1ac650b2b4ed92e8f [ 809.296960] env[64020]: INFO nova.compute.manager [-] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Took 1.03 seconds to deallocate network for instance. [ 809.299305] env[64020]: DEBUG nova.compute.claims [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 809.299477] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 809.342098] env[64020]: ERROR nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e83e0437-9379-4de3-b9fd-44e3086389c7, please check neutron logs for more information. [ 809.342098] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 809.342098] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 809.342098] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 809.342098] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 809.342098] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 809.342098] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 809.342098] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 809.342098] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 809.342098] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 809.342098] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 809.342098] env[64020]: ERROR nova.compute.manager raise self.value [ 809.342098] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 809.342098] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 809.342098] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 809.342098] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 809.342683] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 809.342683] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 809.342683] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e83e0437-9379-4de3-b9fd-44e3086389c7, please check neutron logs for more information. [ 809.342683] env[64020]: ERROR nova.compute.manager [ 809.342683] env[64020]: Traceback (most recent call last): [ 809.342683] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 809.342683] env[64020]: listener.cb(fileno) [ 809.342683] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 809.342683] env[64020]: result = function(*args, **kwargs) [ 809.342683] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 809.342683] env[64020]: return func(*args, **kwargs) [ 809.342683] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 809.342683] env[64020]: raise e [ 809.342683] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 809.342683] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 809.342683] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 809.342683] env[64020]: created_port_ids = self._update_ports_for_instance( [ 809.342683] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 809.342683] env[64020]: with excutils.save_and_reraise_exception(): [ 809.342683] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 809.342683] env[64020]: self.force_reraise() [ 809.342683] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 809.342683] env[64020]: raise self.value [ 809.342683] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 809.342683] env[64020]: updated_port = self._update_port( [ 809.342683] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 809.342683] env[64020]: _ensure_no_port_binding_failure(port) [ 809.342683] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 809.342683] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 809.343637] env[64020]: nova.exception.PortBindingFailed: Binding failed for port e83e0437-9379-4de3-b9fd-44e3086389c7, please check neutron logs for more information. [ 809.343637] env[64020]: Removing descriptor: 18 [ 809.472130] env[64020]: DEBUG nova.scheduler.client.report [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 809.474622] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 7d4d63fdd3854e80b0f41f37157f77e6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 809.489974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d4d63fdd3854e80b0f41f37157f77e6 [ 809.531112] env[64020]: DEBUG nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 809.562981] env[64020]: DEBUG nova.virt.hardware [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 809.563233] env[64020]: DEBUG nova.virt.hardware [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 809.563389] env[64020]: DEBUG nova.virt.hardware [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 809.563569] env[64020]: DEBUG nova.virt.hardware [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 809.563713] env[64020]: DEBUG nova.virt.hardware [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 809.563858] env[64020]: DEBUG nova.virt.hardware [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 809.564076] env[64020]: DEBUG nova.virt.hardware [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 809.564260] env[64020]: DEBUG nova.virt.hardware [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 809.564473] env[64020]: DEBUG nova.virt.hardware [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 809.564641] env[64020]: DEBUG nova.virt.hardware [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 809.564811] env[64020]: DEBUG nova.virt.hardware [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 809.565665] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f17901a-fb93-44b1-9409-d2def0c5a573 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.573688] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8820f947-f4ac-44a9-a15c-863dee491817 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.587268] env[64020]: ERROR nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e83e0437-9379-4de3-b9fd-44e3086389c7, please check neutron logs for more information. [ 809.587268] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Traceback (most recent call last): [ 809.587268] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 809.587268] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] yield resources [ 809.587268] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 809.587268] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] self.driver.spawn(context, instance, image_meta, [ 809.587268] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 809.587268] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] self._vmops.spawn(context, instance, image_meta, injected_files, [ 809.587268] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 809.587268] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] vm_ref = self.build_virtual_machine(instance, [ 809.587268] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] vif_infos = vmwarevif.get_vif_info(self._session, [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] for vif in network_info: [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] return self._sync_wrapper(fn, *args, **kwargs) [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] self.wait() [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] self[:] = self._gt.wait() [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] return self._exit_event.wait() [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 809.587712] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] current.throw(*self._exc) [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] result = function(*args, **kwargs) [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] return func(*args, **kwargs) [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] raise e [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] nwinfo = self.network_api.allocate_for_instance( [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] created_port_ids = self._update_ports_for_instance( [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] with excutils.save_and_reraise_exception(): [ 809.588224] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 809.588679] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] self.force_reraise() [ 809.588679] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 809.588679] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] raise self.value [ 809.588679] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 809.588679] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] updated_port = self._update_port( [ 809.588679] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 809.588679] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] _ensure_no_port_binding_failure(port) [ 809.588679] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 809.588679] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] raise exception.PortBindingFailed(port_id=port['id']) [ 809.588679] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] nova.exception.PortBindingFailed: Binding failed for port e83e0437-9379-4de3-b9fd-44e3086389c7, please check neutron logs for more information. [ 809.588679] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] [ 809.588679] env[64020]: INFO nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Terminating instance [ 809.589629] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquiring lock "refresh_cache-d6228174-ceb1-4ca0-b62b-27db15c20a85" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 809.589788] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquired lock "refresh_cache-d6228174-ceb1-4ca0-b62b-27db15c20a85" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 809.589952] env[64020]: DEBUG nova.network.neutron [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 809.592493] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 835f0c37dd5d4f95bff8af692e948e26 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 809.597547] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 835f0c37dd5d4f95bff8af692e948e26 [ 809.718688] env[64020]: DEBUG oslo_concurrency.lockutils [req-df5ec41f-414e-45e1-9e13-c076cf511a06 req-dacfdbf3-e955-4dee-8134-cb0c5088381c service nova] Releasing lock "refresh_cache-73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 809.718945] env[64020]: DEBUG nova.compute.manager [req-df5ec41f-414e-45e1-9e13-c076cf511a06 req-dacfdbf3-e955-4dee-8134-cb0c5088381c service nova] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Received event network-vif-deleted-bcd04248-654c-44de-ab87-7f1272bc06d3 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 809.979176] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.476s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 809.979176] env[64020]: DEBUG nova.compute.manager [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 809.980820] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg a1f394291b124f67a1a7c04cd48e16d3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 809.981849] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.534s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 809.984977] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 8d75c6e7d92141d2bcbaf10d086f45eb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 810.020117] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1f394291b124f67a1a7c04cd48e16d3 [ 810.057379] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d75c6e7d92141d2bcbaf10d086f45eb [ 810.108061] env[64020]: DEBUG nova.network.neutron [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 810.196481] env[64020]: DEBUG nova.network.neutron [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 810.196481] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 4382ae7fb85c4a038a730dfa8b52dc30 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 810.208624] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4382ae7fb85c4a038a730dfa8b52dc30 [ 810.490502] env[64020]: DEBUG nova.compute.utils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 810.491122] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 4e24642e19944b3684316372aeefa32d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 810.492262] env[64020]: DEBUG nova.compute.manager [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Not allocating networking since 'none' was specified. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 810.504789] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e24642e19944b3684316372aeefa32d [ 810.589308] env[64020]: DEBUG nova.compute.manager [req-d2f080cf-510d-456c-84f0-3c7cb31efa4e req-e24ff0bc-39c6-4a35-90fd-140c281e5513 service nova] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Received event network-changed-e83e0437-9379-4de3-b9fd-44e3086389c7 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 810.589490] env[64020]: DEBUG nova.compute.manager [req-d2f080cf-510d-456c-84f0-3c7cb31efa4e req-e24ff0bc-39c6-4a35-90fd-140c281e5513 service nova] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Refreshing instance network info cache due to event network-changed-e83e0437-9379-4de3-b9fd-44e3086389c7. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 810.589674] env[64020]: DEBUG oslo_concurrency.lockutils [req-d2f080cf-510d-456c-84f0-3c7cb31efa4e req-e24ff0bc-39c6-4a35-90fd-140c281e5513 service nova] Acquiring lock "refresh_cache-d6228174-ceb1-4ca0-b62b-27db15c20a85" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 810.701431] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Releasing lock "refresh_cache-d6228174-ceb1-4ca0-b62b-27db15c20a85" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 810.701848] env[64020]: DEBUG nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 810.702030] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 810.702378] env[64020]: DEBUG oslo_concurrency.lockutils [req-d2f080cf-510d-456c-84f0-3c7cb31efa4e req-e24ff0bc-39c6-4a35-90fd-140c281e5513 service nova] Acquired lock "refresh_cache-d6228174-ceb1-4ca0-b62b-27db15c20a85" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 810.702545] env[64020]: DEBUG nova.network.neutron [req-d2f080cf-510d-456c-84f0-3c7cb31efa4e req-e24ff0bc-39c6-4a35-90fd-140c281e5513 service nova] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Refreshing network info cache for port e83e0437-9379-4de3-b9fd-44e3086389c7 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 810.702964] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-d2f080cf-510d-456c-84f0-3c7cb31efa4e req-e24ff0bc-39c6-4a35-90fd-140c281e5513 service nova] Expecting reply to msg 07b529e598f54d8691a4f3c3ca534aad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 810.704045] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-54dc4edf-4200-4c3f-ba31-cf9be9c4be74 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.716180] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5a497e2-11a4-47fc-a4b0-e2a2ffc7a618 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.726518] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 07b529e598f54d8691a4f3c3ca534aad [ 810.741725] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d6228174-ceb1-4ca0-b62b-27db15c20a85 could not be found. [ 810.741971] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 810.742165] env[64020]: INFO nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Took 0.04 seconds to destroy the instance on the hypervisor. [ 810.742443] env[64020]: DEBUG oslo.service.loopingcall [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 810.744965] env[64020]: DEBUG nova.compute.manager [-] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 810.745067] env[64020]: DEBUG nova.network.neutron [-] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 810.764487] env[64020]: DEBUG nova.network.neutron [-] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 810.765049] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bec201cfb701441fac8fa056888af789 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 810.772674] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bec201cfb701441fac8fa056888af789 [ 810.878060] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-356b0902-1e62-455a-b09f-dc75df8fc44d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.886475] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26d78665-4a8a-4fee-b0dd-2ae18dc97737 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.921336] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40cd2a7a-c59b-4892-8c5c-b2c0afafeadc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.929812] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ca63edd-8b8b-49e1-92c4-8741d5604eb6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.945358] env[64020]: DEBUG nova.compute.provider_tree [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 810.946046] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 4e0f206a5bd1499784d0fb087cc48aaa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 810.953608] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e0f206a5bd1499784d0fb087cc48aaa [ 810.997730] env[64020]: DEBUG nova.compute.manager [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 810.999898] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg d56bd190ddaa4ff0bba052f25caf0abc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 811.032479] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d56bd190ddaa4ff0bba052f25caf0abc [ 811.116274] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 811.116274] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 811.222982] env[64020]: DEBUG nova.network.neutron [req-d2f080cf-510d-456c-84f0-3c7cb31efa4e req-e24ff0bc-39c6-4a35-90fd-140c281e5513 service nova] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 811.270952] env[64020]: DEBUG nova.network.neutron [-] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 811.271585] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c36bbdbd4868403a9ffb1ddf861107f1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 811.280892] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c36bbdbd4868403a9ffb1ddf861107f1 [ 811.300169] env[64020]: DEBUG nova.network.neutron [req-d2f080cf-510d-456c-84f0-3c7cb31efa4e req-e24ff0bc-39c6-4a35-90fd-140c281e5513 service nova] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 811.300759] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-d2f080cf-510d-456c-84f0-3c7cb31efa4e req-e24ff0bc-39c6-4a35-90fd-140c281e5513 service nova] Expecting reply to msg f7b5e18bf4a747248f098b450958bd6e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 811.308620] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7b5e18bf4a747248f098b450958bd6e [ 811.454806] env[64020]: DEBUG nova.scheduler.client.report [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 811.457262] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 8d885446518c48c387e1100b66b6c742 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 811.470212] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d885446518c48c387e1100b66b6c742 [ 811.505157] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 871f70bfb9194f91bf5f10c0a2fceedd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 811.541037] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 871f70bfb9194f91bf5f10c0a2fceedd [ 811.774075] env[64020]: INFO nova.compute.manager [-] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Took 1.03 seconds to deallocate network for instance. [ 811.776346] env[64020]: DEBUG nova.compute.claims [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 811.776519] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 811.803583] env[64020]: DEBUG oslo_concurrency.lockutils [req-d2f080cf-510d-456c-84f0-3c7cb31efa4e req-e24ff0bc-39c6-4a35-90fd-140c281e5513 service nova] Releasing lock "refresh_cache-d6228174-ceb1-4ca0-b62b-27db15c20a85" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 811.803806] env[64020]: DEBUG nova.compute.manager [req-d2f080cf-510d-456c-84f0-3c7cb31efa4e req-e24ff0bc-39c6-4a35-90fd-140c281e5513 service nova] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Received event network-vif-deleted-e83e0437-9379-4de3-b9fd-44e3086389c7 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 811.960166] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.978s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 811.960831] env[64020]: ERROR nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 04a04760-7eee-4ce9-98f4-56ed86f6375d, please check neutron logs for more information. [ 811.960831] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] Traceback (most recent call last): [ 811.960831] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 811.960831] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] self.driver.spawn(context, instance, image_meta, [ 811.960831] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 811.960831] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] self._vmops.spawn(context, instance, image_meta, injected_files, [ 811.960831] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 811.960831] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] vm_ref = self.build_virtual_machine(instance, [ 811.960831] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 811.960831] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] vif_infos = vmwarevif.get_vif_info(self._session, [ 811.960831] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] for vif in network_info: [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] return self._sync_wrapper(fn, *args, **kwargs) [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] self.wait() [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] self[:] = self._gt.wait() [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] return self._exit_event.wait() [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] current.throw(*self._exc) [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 811.961372] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] result = function(*args, **kwargs) [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] return func(*args, **kwargs) [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] raise e [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] nwinfo = self.network_api.allocate_for_instance( [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] created_port_ids = self._update_ports_for_instance( [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] with excutils.save_and_reraise_exception(): [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] self.force_reraise() [ 811.961773] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 811.962180] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] raise self.value [ 811.962180] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 811.962180] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] updated_port = self._update_port( [ 811.962180] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 811.962180] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] _ensure_no_port_binding_failure(port) [ 811.962180] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 811.962180] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] raise exception.PortBindingFailed(port_id=port['id']) [ 811.962180] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] nova.exception.PortBindingFailed: Binding failed for port 04a04760-7eee-4ce9-98f4-56ed86f6375d, please check neutron logs for more information. [ 811.962180] env[64020]: ERROR nova.compute.manager [instance: f873c026-0fa1-4176-a30f-207767f87410] [ 811.962180] env[64020]: DEBUG nova.compute.utils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Binding failed for port 04a04760-7eee-4ce9-98f4-56ed86f6375d, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 811.962998] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.827s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 811.965088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 3e220c376375499c9dc739904eca6569 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 811.966469] env[64020]: DEBUG nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Build of instance f873c026-0fa1-4176-a30f-207767f87410 was re-scheduled: Binding failed for port 04a04760-7eee-4ce9-98f4-56ed86f6375d, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 811.967060] env[64020]: DEBUG nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 811.967309] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Acquiring lock "refresh_cache-f873c026-0fa1-4176-a30f-207767f87410" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 811.967496] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Acquired lock "refresh_cache-f873c026-0fa1-4176-a30f-207767f87410" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 811.967672] env[64020]: DEBUG nova.network.neutron [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 811.968155] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 4e04183922414fe683d2637970a7379b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 811.975488] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e04183922414fe683d2637970a7379b [ 811.997660] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e220c376375499c9dc739904eca6569 [ 812.008740] env[64020]: DEBUG nova.compute.manager [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 812.035364] env[64020]: DEBUG nova.virt.hardware [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 812.035592] env[64020]: DEBUG nova.virt.hardware [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 812.035739] env[64020]: DEBUG nova.virt.hardware [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 812.035909] env[64020]: DEBUG nova.virt.hardware [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 812.036060] env[64020]: DEBUG nova.virt.hardware [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 812.036213] env[64020]: DEBUG nova.virt.hardware [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 812.036415] env[64020]: DEBUG nova.virt.hardware [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 812.036561] env[64020]: DEBUG nova.virt.hardware [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 812.036715] env[64020]: DEBUG nova.virt.hardware [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 812.036866] env[64020]: DEBUG nova.virt.hardware [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 812.037023] env[64020]: DEBUG nova.virt.hardware [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 812.037867] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-adceda44-409d-465d-a9f3-95ac7bafe0ae {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.046381] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c413c42-e4b2-494d-88b9-807d288374a9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.060103] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 812.065737] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Creating folder: Project (e9f9a8f30e794971922560c7cf63c07f). Parent ref: group-v110249. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 812.066368] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-712a7810-62e7-4334-bd2d-5b4f9153e4ce {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.077365] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Created folder: Project (e9f9a8f30e794971922560c7cf63c07f) in parent group-v110249. [ 812.077559] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Creating folder: Instances. Parent ref: group-v110263. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 812.077842] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-5a6cd87a-dc75-4743-a3fd-2d5a22209208 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.086668] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Created folder: Instances in parent group-v110263. [ 812.086914] env[64020]: DEBUG oslo.service.loopingcall [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 812.087104] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 812.087301] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-82bc6c91-2fa6-4709-ac68-586c845bd899 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.105515] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 812.105515] env[64020]: value = "task-407874" [ 812.105515] env[64020]: _type = "Task" [ 812.105515] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 812.112681] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407874, 'name': CreateVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 812.487002] env[64020]: DEBUG nova.network.neutron [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 812.572130] env[64020]: DEBUG nova.network.neutron [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 812.572627] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg e9defc1a478845da9be0c5968f0c8255 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 812.582467] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9defc1a478845da9be0c5968f0c8255 [ 812.616866] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407874, 'name': CreateVM_Task, 'duration_secs': 0.226838} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 812.617283] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 812.617721] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 812.617873] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 812.618217] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 812.618486] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b1f49921-e831-4c56-9cd9-51ce63c47995 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.624891] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 812.624891] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52b77c31-34d2-b795-ea49-813c17580a59" [ 812.624891] env[64020]: _type = "Task" [ 812.624891] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 812.634396] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52b77c31-34d2-b795-ea49-813c17580a59, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 812.755683] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5aa1ea7e-6c76-4e4f-b92c-66c73b53467f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.762864] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e3c7e3d-2afe-4347-95dd-90a512764037 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.791472] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-803d661f-51b5-49f6-ac73-d949c4fb2371 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.798421] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0953be9-14d6-47bc-a6ef-396719d8a282 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.811201] env[64020]: DEBUG nova.compute.provider_tree [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 812.811701] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 7472a57ab1ed4fe19638104cea5563f4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 812.822644] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7472a57ab1ed4fe19638104cea5563f4 [ 813.074939] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Releasing lock "refresh_cache-f873c026-0fa1-4176-a30f-207767f87410" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 813.075366] env[64020]: DEBUG nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 813.075366] env[64020]: DEBUG nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 813.075502] env[64020]: DEBUG nova.network.neutron [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 813.091120] env[64020]: DEBUG nova.network.neutron [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 813.091762] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 5a1d12d6c0ab47d09f54cc3f1107d21a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 813.099061] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a1d12d6c0ab47d09f54cc3f1107d21a [ 813.134916] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52b77c31-34d2-b795-ea49-813c17580a59, 'name': SearchDatastore_Task, 'duration_secs': 0.009669} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 813.135230] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 813.135457] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 813.135680] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 813.135820] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 813.135992] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 813.136254] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-77a525f7-c6b2-4835-8bff-3af2e2cd2202 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.143727] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 813.143894] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 813.144601] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-64acdff8-5e99-4644-acb1-296aa13af4e6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.149872] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 813.149872] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52846c27-a1a8-a58a-3f3a-5e1299394d3e" [ 813.149872] env[64020]: _type = "Task" [ 813.149872] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 813.157053] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52846c27-a1a8-a58a-3f3a-5e1299394d3e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 813.314894] env[64020]: DEBUG nova.scheduler.client.report [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 813.317321] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 4a6c057a3f8b4f92892a85510ebfd4d1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 813.330670] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4a6c057a3f8b4f92892a85510ebfd4d1 [ 813.594408] env[64020]: DEBUG nova.network.neutron [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 813.594930] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg b5afb554eb1643d886eec108c9719e2b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 813.603240] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b5afb554eb1643d886eec108c9719e2b [ 813.660929] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52846c27-a1a8-a58a-3f3a-5e1299394d3e, 'name': SearchDatastore_Task, 'duration_secs': 0.007604} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 813.661707] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-732be461-f65e-4fee-91ba-fe1a74679c69 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.666475] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 813.666475] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5227cc04-8788-e6e7-c150-cfa286e4738e" [ 813.666475] env[64020]: _type = "Task" [ 813.666475] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 813.674026] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5227cc04-8788-e6e7-c150-cfa286e4738e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 813.819781] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.857s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 813.820524] env[64020]: ERROR nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 02caced1-357c-4472-81cc-925703a5e48b, please check neutron logs for more information. [ 813.820524] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Traceback (most recent call last): [ 813.820524] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 813.820524] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] self.driver.spawn(context, instance, image_meta, [ 813.820524] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 813.820524] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 813.820524] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 813.820524] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] vm_ref = self.build_virtual_machine(instance, [ 813.820524] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 813.820524] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] vif_infos = vmwarevif.get_vif_info(self._session, [ 813.820524] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] for vif in network_info: [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] return self._sync_wrapper(fn, *args, **kwargs) [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] self.wait() [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] self[:] = self._gt.wait() [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] return self._exit_event.wait() [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] current.throw(*self._exc) [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 813.820924] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] result = function(*args, **kwargs) [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] return func(*args, **kwargs) [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] raise e [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] nwinfo = self.network_api.allocate_for_instance( [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] created_port_ids = self._update_ports_for_instance( [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] with excutils.save_and_reraise_exception(): [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] self.force_reraise() [ 813.821367] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 813.821797] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] raise self.value [ 813.821797] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 813.821797] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] updated_port = self._update_port( [ 813.821797] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 813.821797] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] _ensure_no_port_binding_failure(port) [ 813.821797] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 813.821797] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] raise exception.PortBindingFailed(port_id=port['id']) [ 813.821797] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] nova.exception.PortBindingFailed: Binding failed for port 02caced1-357c-4472-81cc-925703a5e48b, please check neutron logs for more information. [ 813.821797] env[64020]: ERROR nova.compute.manager [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] [ 813.821797] env[64020]: DEBUG nova.compute.utils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Binding failed for port 02caced1-357c-4472-81cc-925703a5e48b, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 813.822524] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 16.423s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 813.822694] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 813.822839] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=64020) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 813.823110] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.285s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 813.825112] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 69ee495ed4514561814cf08b2130e069 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 813.826265] env[64020]: DEBUG nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Build of instance 052ae9f0-bb03-4747-b8f9-708e4a7c147c was re-scheduled: Binding failed for port 02caced1-357c-4472-81cc-925703a5e48b, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 813.826689] env[64020]: DEBUG nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 813.826910] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Acquiring lock "refresh_cache-052ae9f0-bb03-4747-b8f9-708e4a7c147c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 813.827046] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Acquired lock "refresh_cache-052ae9f0-bb03-4747-b8f9-708e4a7c147c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 813.827199] env[64020]: DEBUG nova.network.neutron [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 813.827557] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg bf4d871bd7aa48c38ba4a77f26f1b822 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 813.834300] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5cd15c22-7920-426a-a063-fd0fba4fd72d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.837276] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf4d871bd7aa48c38ba4a77f26f1b822 [ 813.843597] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7a75382-e31a-4ea8-bf78-a91f8ba5f12d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.857929] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4d043b0-b1ad-4cef-a24f-d945907fb07f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.864879] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea97df9e-3f1a-4880-82b3-e8e3a011c32b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.893291] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69ee495ed4514561814cf08b2130e069 [ 813.893790] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181807MB free_disk=120GB free_vcpus=48 pci_devices=None {{(pid=64020) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 813.893934] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 814.096971] env[64020]: INFO nova.compute.manager [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] [instance: f873c026-0fa1-4176-a30f-207767f87410] Took 1.02 seconds to deallocate network for instance. [ 814.099204] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg 389876c61c4e4ae8893eea8338b55b9a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 814.130851] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 389876c61c4e4ae8893eea8338b55b9a [ 814.177311] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5227cc04-8788-e6e7-c150-cfa286e4738e, 'name': SearchDatastore_Task, 'duration_secs': 0.007454} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 814.177735] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 814.178096] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] e27f988a-a64b-407b-ad66-4bed7ee45b52/e27f988a-a64b-407b-ad66-4bed7ee45b52.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 814.178496] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-a47c1b43-6172-42d0-82a8-409a718a4bda {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.185589] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 814.185589] env[64020]: value = "task-407875" [ 814.185589] env[64020]: _type = "Task" [ 814.185589] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 814.195723] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407875, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 814.364766] env[64020]: DEBUG nova.network.neutron [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 814.459362] env[64020]: DEBUG nova.network.neutron [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 814.459362] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 7ee2e661901e4b84a70d4dd5be75b350 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 814.466961] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ee2e661901e4b84a70d4dd5be75b350 [ 814.606643] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg e7608cace59e4b08a30678bd9ca274ad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 814.645740] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e7608cace59e4b08a30678bd9ca274ad [ 814.697855] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407875, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.469464} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 814.698117] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] e27f988a-a64b-407b-ad66-4bed7ee45b52/e27f988a-a64b-407b-ad66-4bed7ee45b52.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 814.698323] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 814.698677] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-12c80a5c-5bf7-487c-ada9-6741cd584bca {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.705006] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 814.705006] env[64020]: value = "task-407876" [ 814.705006] env[64020]: _type = "Task" [ 814.705006] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 814.715258] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407876, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 814.751831] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-387ba029-bb41-4827-a024-caa1ba97e030 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.758564] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e82c4758-8977-4496-8fb3-9fbd6c062f4f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.789841] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e940ca0-e981-4a64-ba99-a3a12ef7af79 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.797808] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f298bff9-ac20-491b-9d44-1fdf9428ca8a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.819084] env[64020]: DEBUG nova.compute.provider_tree [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 814.819605] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 35b6b945f730435cba5e92e943b37e9a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 814.826636] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35b6b945f730435cba5e92e943b37e9a [ 814.961359] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Releasing lock "refresh_cache-052ae9f0-bb03-4747-b8f9-708e4a7c147c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 814.961622] env[64020]: DEBUG nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 814.961805] env[64020]: DEBUG nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 814.961969] env[64020]: DEBUG nova.network.neutron [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 814.976811] env[64020]: DEBUG nova.network.neutron [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 814.977362] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 0f71e817480544099a0602c6d092f7bb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 814.988799] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f71e817480544099a0602c6d092f7bb [ 815.139101] env[64020]: INFO nova.scheduler.client.report [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Deleted allocations for instance f873c026-0fa1-4176-a30f-207767f87410 [ 815.145270] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Expecting reply to msg ad3aac0b38f145aea1d5fe42295926d0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 815.163746] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad3aac0b38f145aea1d5fe42295926d0 [ 815.214599] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407876, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.058599} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 815.214863] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 815.215612] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fcb23e99-59be-42dd-986a-fe1a81a47c7b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 815.234839] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Reconfiguring VM instance instance-00000031 to attach disk [datastore1] e27f988a-a64b-407b-ad66-4bed7ee45b52/e27f988a-a64b-407b-ad66-4bed7ee45b52.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 815.235381] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-afe1ea32-8d71-4dab-a2bf-a1b0e4478568 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 815.255166] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 815.255166] env[64020]: value = "task-407877" [ 815.255166] env[64020]: _type = "Task" [ 815.255166] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 815.262964] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407877, 'name': ReconfigVM_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 815.322344] env[64020]: DEBUG nova.scheduler.client.report [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 815.325654] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg e233bd68fb364f5a9b380f372c93e39f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 815.340127] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e233bd68fb364f5a9b380f372c93e39f [ 815.479855] env[64020]: DEBUG nova.network.neutron [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 815.480399] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 7632732ff968469987e6b8135046e440 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 815.489119] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7632732ff968469987e6b8135046e440 [ 815.646789] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8a1cffc-67a6-4664-ab4a-f20b8936b6e3 tempest-ServersTestManualDisk-1151884279 tempest-ServersTestManualDisk-1151884279-project-member] Lock "f873c026-0fa1-4176-a30f-207767f87410" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 175.840s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 815.647394] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 3e774a81b9bf4e38a353869c1d31fdcd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 815.657252] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e774a81b9bf4e38a353869c1d31fdcd [ 815.765614] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407877, 'name': ReconfigVM_Task, 'duration_secs': 0.287413} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 815.765884] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Reconfigured VM instance instance-00000031 to attach disk [datastore1] e27f988a-a64b-407b-ad66-4bed7ee45b52/e27f988a-a64b-407b-ad66-4bed7ee45b52.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 815.766439] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-27f8c87c-320c-4314-ab98-535ab0ca6421 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 815.772581] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 815.772581] env[64020]: value = "task-407878" [ 815.772581] env[64020]: _type = "Task" [ 815.772581] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 815.780148] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407878, 'name': Rename_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 815.828649] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.005s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 815.829414] env[64020]: ERROR nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 417cecbc-3d8a-4119-b90d-38716e59bc58, please check neutron logs for more information. [ 815.829414] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Traceback (most recent call last): [ 815.829414] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 815.829414] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] self.driver.spawn(context, instance, image_meta, [ 815.829414] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 815.829414] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 815.829414] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 815.829414] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] vm_ref = self.build_virtual_machine(instance, [ 815.829414] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 815.829414] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] vif_infos = vmwarevif.get_vif_info(self._session, [ 815.829414] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] for vif in network_info: [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] return self._sync_wrapper(fn, *args, **kwargs) [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] self.wait() [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] self[:] = self._gt.wait() [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] return self._exit_event.wait() [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] current.throw(*self._exc) [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 815.829851] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] result = function(*args, **kwargs) [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] return func(*args, **kwargs) [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] raise e [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] nwinfo = self.network_api.allocate_for_instance( [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] created_port_ids = self._update_ports_for_instance( [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] with excutils.save_and_reraise_exception(): [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] self.force_reraise() [ 815.830319] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 815.830760] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] raise self.value [ 815.830760] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 815.830760] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] updated_port = self._update_port( [ 815.830760] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 815.830760] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] _ensure_no_port_binding_failure(port) [ 815.830760] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 815.830760] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] raise exception.PortBindingFailed(port_id=port['id']) [ 815.830760] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] nova.exception.PortBindingFailed: Binding failed for port 417cecbc-3d8a-4119-b90d-38716e59bc58, please check neutron logs for more information. [ 815.830760] env[64020]: ERROR nova.compute.manager [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] [ 815.830760] env[64020]: DEBUG nova.compute.utils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Binding failed for port 417cecbc-3d8a-4119-b90d-38716e59bc58, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 815.831489] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.817s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 815.833506] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 665687ee0725425ead52c24bdd6208f6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 815.834639] env[64020]: DEBUG nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Build of instance b89f34cd-bd7b-4f22-8e6f-a167d9f01af8 was re-scheduled: Binding failed for port 417cecbc-3d8a-4119-b90d-38716e59bc58, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 815.835043] env[64020]: DEBUG nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 815.835293] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Acquiring lock "refresh_cache-b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 815.835442] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Acquired lock "refresh_cache-b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 815.835599] env[64020]: DEBUG nova.network.neutron [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 815.835947] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg a00d9d02549e4b70916dba2bc2740f59 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 815.845944] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a00d9d02549e4b70916dba2bc2740f59 [ 815.863359] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 665687ee0725425ead52c24bdd6208f6 [ 815.983240] env[64020]: INFO nova.compute.manager [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] [instance: 052ae9f0-bb03-4747-b8f9-708e4a7c147c] Took 1.02 seconds to deallocate network for instance. [ 815.985173] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 06e8c50361e24954a01503e1adae7291 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 816.026256] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06e8c50361e24954a01503e1adae7291 [ 816.150379] env[64020]: DEBUG nova.compute.manager [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 816.152218] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg d5c8f24c08fe43f2aac846f666bdd24e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 816.184903] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5c8f24c08fe43f2aac846f666bdd24e [ 816.283175] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407878, 'name': Rename_Task, 'duration_secs': 0.152744} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 816.283450] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 816.283753] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-7619b39f-c451-47c7-bba3-8e83025fdcb4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.289705] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 816.289705] env[64020]: value = "task-407879" [ 816.289705] env[64020]: _type = "Task" [ 816.289705] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 816.298457] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407879, 'name': PowerOnVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 816.358913] env[64020]: DEBUG nova.network.neutron [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 816.491468] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 0c61a30067874400b3768e9c249cc9cb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 816.528615] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c61a30067874400b3768e9c249cc9cb [ 816.603995] env[64020]: DEBUG nova.network.neutron [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 816.604412] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg d5d4f916744748a5a392c31f68de5ad4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 816.623382] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5d4f916744748a5a392c31f68de5ad4 [ 816.671034] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 816.673016] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-532076bb-5be4-4983-bc44-2faf37dba4a7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.680495] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd51d0ce-e4cc-454e-9428-68f0221d67a6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.715982] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2914aff3-9448-4946-b97f-065ce9d95973 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.723611] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ed00ff4-6903-470c-afc7-bc8b07e130f1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.736492] env[64020]: DEBUG nova.compute.provider_tree [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 816.737001] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 8ec14bde75d148609f676dba519bdb96 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 816.745240] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ec14bde75d148609f676dba519bdb96 [ 816.799644] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407879, 'name': PowerOnVM_Task} progress is 66%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 817.017941] env[64020]: INFO nova.scheduler.client.report [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Deleted allocations for instance 052ae9f0-bb03-4747-b8f9-708e4a7c147c [ 817.024514] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Expecting reply to msg 6423e52dc23545a7a821731863bac937 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 817.038090] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6423e52dc23545a7a821731863bac937 [ 817.106525] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Releasing lock "refresh_cache-b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 817.106756] env[64020]: DEBUG nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 817.106935] env[64020]: DEBUG nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 817.107098] env[64020]: DEBUG nova.network.neutron [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 817.123120] env[64020]: DEBUG nova.network.neutron [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 817.124028] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 52ede32792534d82ad55987b336c7720 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 817.131096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52ede32792534d82ad55987b336c7720 [ 817.239948] env[64020]: DEBUG nova.scheduler.client.report [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 817.242473] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg d9465874acbd436391294b66fa5d289f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 817.256420] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9465874acbd436391294b66fa5d289f [ 817.300765] env[64020]: DEBUG oslo_vmware.api [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407879, 'name': PowerOnVM_Task, 'duration_secs': 0.780257} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 817.301020] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 817.301271] env[64020]: INFO nova.compute.manager [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Took 5.29 seconds to spawn the instance on the hypervisor. [ 817.301458] env[64020]: DEBUG nova.compute.manager [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 817.303411] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00ef393e-f533-4204-a331-fce54b7092d0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.309607] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg e52e39c0ab944a269af7f278ee95a8b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 817.344880] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e52e39c0ab944a269af7f278ee95a8b3 [ 817.526984] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c7500daa-88eb-4635-bc7e-21b8f067bf7e tempest-InstanceActionsV221TestJSON-567743484 tempest-InstanceActionsV221TestJSON-567743484-project-member] Lock "052ae9f0-bb03-4747-b8f9-708e4a7c147c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 175.503s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.527419] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg f7aee9616fa54183a0956a7dae42e917 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 817.537805] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7aee9616fa54183a0956a7dae42e917 [ 817.627905] env[64020]: DEBUG nova.network.neutron [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 817.628471] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 0fae26c89d0d4cf6b8f665178e0364e5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 817.641937] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0fae26c89d0d4cf6b8f665178e0364e5 [ 817.746569] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.913s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.746569] env[64020]: ERROR nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4dba8130-7cfd-4940-86fb-303a6f32e278, please check neutron logs for more information. [ 817.746569] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] Traceback (most recent call last): [ 817.746569] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 817.746569] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] self.driver.spawn(context, instance, image_meta, [ 817.746569] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 817.746569] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] self._vmops.spawn(context, instance, image_meta, injected_files, [ 817.746569] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 817.746569] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] vm_ref = self.build_virtual_machine(instance, [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] vif_infos = vmwarevif.get_vif_info(self._session, [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] for vif in network_info: [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] return self._sync_wrapper(fn, *args, **kwargs) [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] self.wait() [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] self[:] = self._gt.wait() [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] return self._exit_event.wait() [ 817.747340] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] current.throw(*self._exc) [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] result = function(*args, **kwargs) [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] return func(*args, **kwargs) [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] raise e [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] nwinfo = self.network_api.allocate_for_instance( [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] created_port_ids = self._update_ports_for_instance( [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 817.747792] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] with excutils.save_and_reraise_exception(): [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] self.force_reraise() [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] raise self.value [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] updated_port = self._update_port( [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] _ensure_no_port_binding_failure(port) [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] raise exception.PortBindingFailed(port_id=port['id']) [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] nova.exception.PortBindingFailed: Binding failed for port 4dba8130-7cfd-4940-86fb-303a6f32e278, please check neutron logs for more information. [ 817.748414] env[64020]: ERROR nova.compute.manager [instance: 1954e745-783e-417e-aafc-265da55066cf] [ 817.748820] env[64020]: DEBUG nova.compute.utils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Binding failed for port 4dba8130-7cfd-4940-86fb-303a6f32e278, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 817.748820] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.982s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 817.749657] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg e50bd362589b4c49b13612cb6c0d90a9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 817.751364] env[64020]: DEBUG nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Build of instance 1954e745-783e-417e-aafc-265da55066cf was re-scheduled: Binding failed for port 4dba8130-7cfd-4940-86fb-303a6f32e278, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 817.751805] env[64020]: DEBUG nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 817.752057] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Acquiring lock "refresh_cache-1954e745-783e-417e-aafc-265da55066cf" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 817.752234] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Acquired lock "refresh_cache-1954e745-783e-417e-aafc-265da55066cf" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 817.752400] env[64020]: DEBUG nova.network.neutron [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 817.752774] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 583712131eb1443981da0309ea1ab751 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 817.759337] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 583712131eb1443981da0309ea1ab751 [ 817.789007] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e50bd362589b4c49b13612cb6c0d90a9 [ 817.817904] env[64020]: INFO nova.compute.manager [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Took 27.14 seconds to build instance. [ 817.818246] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 4baddc6b699640ecad337fdee0296d8c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 817.833307] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4baddc6b699640ecad337fdee0296d8c [ 818.033402] env[64020]: DEBUG nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 818.035155] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg dc90208ba08b46acbc66838f1137133f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 818.072545] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg d9eaec515b2242f2aca29a97ac893600 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 818.073816] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc90208ba08b46acbc66838f1137133f [ 818.094089] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9eaec515b2242f2aca29a97ac893600 [ 818.131252] env[64020]: INFO nova.compute.manager [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] [instance: b89f34cd-bd7b-4f22-8e6f-a167d9f01af8] Took 1.02 seconds to deallocate network for instance. [ 818.133126] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg b999e5a7034e4b7fa10cc71e41962aec in queue reply_57893177120949e6a93cb88e15cd42b4 [ 818.181578] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b999e5a7034e4b7fa10cc71e41962aec [ 818.273139] env[64020]: DEBUG nova.network.neutron [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 818.321115] env[64020]: DEBUG oslo_concurrency.lockutils [None req-602be63c-0c71-45c2-b52d-1efe354a9510 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "e27f988a-a64b-407b-ad66-4bed7ee45b52" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 132.107s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 818.329469] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 41ead20b4bfa4eec9999ef5274778fd5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 818.346043] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 41ead20b4bfa4eec9999ef5274778fd5 [ 818.371503] env[64020]: DEBUG nova.network.neutron [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 818.372019] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg dead6eee6b494ec7834107023af53bb1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 818.383037] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dead6eee6b494ec7834107023af53bb1 [ 818.560654] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 818.575214] env[64020]: INFO nova.compute.manager [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Rebuilding instance [ 818.609536] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-441c5c40-abac-4091-93a7-5e702ac46cae {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.620753] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a547fa91-a814-4400-a199-a653681367cb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.628062] env[64020]: DEBUG nova.compute.manager [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 818.628831] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1af8fb62-9fe4-4fdc-8eca-2c50a8a72e9f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.660649] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg 09dd4dbd52b64d5b907b6fa710c96cd6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 818.663185] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9cf7695-20c6-418b-bcf4-ecd69224be86 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.670973] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg dee1da9bd162451aa1ed68fad1749d17 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 818.675238] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c66a6295-824d-40d3-af70-c4cc8e634d21 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.691804] env[64020]: DEBUG nova.compute.provider_tree [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 818.708518] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 573cb48acf3647068344934cb366807f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 818.716825] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 573cb48acf3647068344934cb366807f [ 818.717411] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dee1da9bd162451aa1ed68fad1749d17 [ 818.725971] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 09dd4dbd52b64d5b907b6fa710c96cd6 [ 818.832057] env[64020]: DEBUG nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 818.834178] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg ba77700571d643ecb6aad724e6d7064c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 818.867172] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba77700571d643ecb6aad724e6d7064c [ 818.875889] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Releasing lock "refresh_cache-1954e745-783e-417e-aafc-265da55066cf" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 818.875889] env[64020]: DEBUG nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 818.875889] env[64020]: DEBUG nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 818.875889] env[64020]: DEBUG nova.network.neutron [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 818.898220] env[64020]: DEBUG nova.network.neutron [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 818.898971] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 35771f5b0a8c4622b3052d30d73ce91c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 818.906375] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35771f5b0a8c4622b3052d30d73ce91c [ 819.177598] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 819.177965] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-d13e7396-02ac-4034-bc81-c514436f8b91 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.186187] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 819.186187] env[64020]: value = "task-407880" [ 819.186187] env[64020]: _type = "Task" [ 819.186187] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 819.195352] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407880, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 819.198408] env[64020]: INFO nova.scheduler.client.report [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Deleted allocations for instance b89f34cd-bd7b-4f22-8e6f-a167d9f01af8 [ 819.204719] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Expecting reply to msg f8ab91226f0f4bf5b27b64af69db2969 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 819.210870] env[64020]: DEBUG nova.scheduler.client.report [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 819.213735] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 52efd912dab84406be68d0a0392c84d5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 819.222725] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f8ab91226f0f4bf5b27b64af69db2969 [ 819.240996] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52efd912dab84406be68d0a0392c84d5 [ 819.358524] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 819.401805] env[64020]: DEBUG nova.network.neutron [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 819.402381] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 011300ce3d864bbca8c3be68614dd1eb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 819.411651] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 011300ce3d864bbca8c3be68614dd1eb [ 819.696040] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407880, 'name': PowerOffVM_Task, 'duration_secs': 0.221672} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 819.696383] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 819.696637] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 819.697430] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a8bb255-9923-4f66-94fb-53c48a0c4fd0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.706259] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 819.706564] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-1d645a52-77ee-45c0-80b5-484946faa9c7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.710896] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f4f54e89-4620-4a4f-88b8-962db6af5855 tempest-ServerActionsV293TestJSON-1320759888 tempest-ServerActionsV293TestJSON-1320759888-project-member] Lock "b89f34cd-bd7b-4f22-8e6f-a167d9f01af8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 175.265s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 819.711537] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg b8dce2a2ef2748bba7df662f5c7c127b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 819.716096] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.969s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 819.719058] env[64020]: ERROR nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 97cd1637-d9ff-46b3-a34d-7ec120db0ffc, please check neutron logs for more information. [ 819.719058] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Traceback (most recent call last): [ 819.719058] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 819.719058] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] self.driver.spawn(context, instance, image_meta, [ 819.719058] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 819.719058] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] self._vmops.spawn(context, instance, image_meta, injected_files, [ 819.719058] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 819.719058] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] vm_ref = self.build_virtual_machine(instance, [ 819.719058] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 819.719058] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] vif_infos = vmwarevif.get_vif_info(self._session, [ 819.719058] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] for vif in network_info: [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] return self._sync_wrapper(fn, *args, **kwargs) [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] self.wait() [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] self[:] = self._gt.wait() [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] return self._exit_event.wait() [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] current.throw(*self._exc) [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 819.719482] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] result = function(*args, **kwargs) [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] return func(*args, **kwargs) [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] raise e [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] nwinfo = self.network_api.allocate_for_instance( [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] created_port_ids = self._update_ports_for_instance( [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] with excutils.save_and_reraise_exception(): [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] self.force_reraise() [ 819.719904] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 819.720349] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] raise self.value [ 819.720349] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 819.720349] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] updated_port = self._update_port( [ 819.720349] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 819.720349] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] _ensure_no_port_binding_failure(port) [ 819.720349] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 819.720349] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] raise exception.PortBindingFailed(port_id=port['id']) [ 819.720349] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] nova.exception.PortBindingFailed: Binding failed for port 97cd1637-d9ff-46b3-a34d-7ec120db0ffc, please check neutron logs for more information. [ 819.720349] env[64020]: ERROR nova.compute.manager [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] [ 819.721372] env[64020]: DEBUG nova.compute.utils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Binding failed for port 97cd1637-d9ff-46b3-a34d-7ec120db0ffc, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 819.723404] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.844s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 819.725417] env[64020]: INFO nova.compute.claims [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 819.727401] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg d1d50166dd8147799a8865a03313fd8f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 819.737032] env[64020]: DEBUG nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Build of instance 5ccefb28-25a8-4876-a31c-58c7edfc9570 was re-scheduled: Binding failed for port 97cd1637-d9ff-46b3-a34d-7ec120db0ffc, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 819.737032] env[64020]: DEBUG nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 819.737032] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Acquiring lock "refresh_cache-5ccefb28-25a8-4876-a31c-58c7edfc9570" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 819.737032] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Acquired lock "refresh_cache-5ccefb28-25a8-4876-a31c-58c7edfc9570" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 819.737370] env[64020]: DEBUG nova.network.neutron [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 819.737370] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg bc1f06baa84e4b74ac84bc02da257d87 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 819.737370] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 819.737370] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Deleting contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 819.737370] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Deleting the datastore file [datastore1] e27f988a-a64b-407b-ad66-4bed7ee45b52 {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 819.737571] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-1e035426-a266-4975-aad2-84c0c2fbbf3c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.740125] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8dce2a2ef2748bba7df662f5c7c127b [ 819.742332] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 819.742332] env[64020]: value = "task-407882" [ 819.742332] env[64020]: _type = "Task" [ 819.742332] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 819.747070] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc1f06baa84e4b74ac84bc02da257d87 [ 819.753004] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407882, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 819.768096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1d50166dd8147799a8865a03313fd8f [ 819.904664] env[64020]: INFO nova.compute.manager [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] [instance: 1954e745-783e-417e-aafc-265da55066cf] Took 1.03 seconds to deallocate network for instance. [ 819.906621] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg 9ce9b950a4fa491498a033517709301a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 819.948375] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ce9b950a4fa491498a033517709301a [ 820.224695] env[64020]: DEBUG nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 820.226592] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg a373a339ae5e4139ad2fad6b1be0729d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 820.233937] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 5c071a0f8994466986537976a7bf97f3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 820.248039] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5c071a0f8994466986537976a7bf97f3 [ 820.260386] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407882, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.098524} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 820.260724] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 820.260908] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Deleted contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 820.261075] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 820.262876] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg a29dcf2591154a75bd9c40c3d38cfb02 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 820.274672] env[64020]: DEBUG nova.network.neutron [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 820.295179] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a373a339ae5e4139ad2fad6b1be0729d [ 820.316034] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a29dcf2591154a75bd9c40c3d38cfb02 [ 820.369698] env[64020]: DEBUG nova.network.neutron [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 820.370480] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 2d6f4f1321034591964678695742136b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 820.378226] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d6f4f1321034591964678695742136b [ 820.411628] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg bda943873e2d4809b6fcadc97974ff41 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 820.458002] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bda943873e2d4809b6fcadc97974ff41 [ 820.756141] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.774486] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 7e32ed6819c14041839a939c8137b83e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 820.817025] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7e32ed6819c14041839a939c8137b83e [ 820.873027] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Releasing lock "refresh_cache-5ccefb28-25a8-4876-a31c-58c7edfc9570" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 820.873232] env[64020]: DEBUG nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 820.873488] env[64020]: DEBUG nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 820.873671] env[64020]: DEBUG nova.network.neutron [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 820.932259] env[64020]: DEBUG nova.network.neutron [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 820.933362] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg e23a449e29c94982b54b492e29e226e7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 820.944435] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e23a449e29c94982b54b492e29e226e7 [ 820.946793] env[64020]: INFO nova.scheduler.client.report [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Deleted allocations for instance 1954e745-783e-417e-aafc-265da55066cf [ 820.960868] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Expecting reply to msg d44db5037f6940ac88f01b027ce5a1cd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 820.979816] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d44db5037f6940ac88f01b027ce5a1cd [ 821.173314] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d183a8b3-9cb8-4e77-9eea-097066668c0b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.181880] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53c2d4a6-7cdd-4005-a38b-41ce7d77a99c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.215482] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ea9c2f2-6d8a-4f64-b141-5278188fc910 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.224263] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e16fe8b-ca0f-4b66-b4dc-2db0457194ba {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.240989] env[64020]: DEBUG nova.compute.provider_tree [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 821.241759] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 0106e35221c9421594265af99bdb9f82 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 821.249968] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0106e35221c9421594265af99bdb9f82 [ 821.320097] env[64020]: DEBUG nova.virt.hardware [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 821.320597] env[64020]: DEBUG nova.virt.hardware [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 821.320759] env[64020]: DEBUG nova.virt.hardware [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 821.320939] env[64020]: DEBUG nova.virt.hardware [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 821.322505] env[64020]: DEBUG nova.virt.hardware [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 821.322719] env[64020]: DEBUG nova.virt.hardware [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 821.322950] env[64020]: DEBUG nova.virt.hardware [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 821.323114] env[64020]: DEBUG nova.virt.hardware [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 821.323283] env[64020]: DEBUG nova.virt.hardware [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 821.323447] env[64020]: DEBUG nova.virt.hardware [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 821.323766] env[64020]: DEBUG nova.virt.hardware [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 821.325003] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2261a0dc-cdd4-4c37-b437-eb56bd3a8d87 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.339162] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2034f47a-28fc-4374-b871-5601e90bcc56 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.357314] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 821.364032] env[64020]: DEBUG oslo.service.loopingcall [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 821.364358] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 821.364579] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-e59c44ee-136b-4497-976d-23f3fd94d3d2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.385872] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 821.385872] env[64020]: value = "task-407884" [ 821.385872] env[64020]: _type = "Task" [ 821.385872] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 821.395836] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407884, 'name': CreateVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 821.437298] env[64020]: DEBUG nova.network.neutron [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 821.437827] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 8101d91940be477f8e5d7dd333e405ee in queue reply_57893177120949e6a93cb88e15cd42b4 [ 821.448625] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8101d91940be477f8e5d7dd333e405ee [ 821.462573] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9ac6c05e-457d-4c4c-8db5-44c778d964b4 tempest-InstanceActionsNegativeTestJSON-802831167 tempest-InstanceActionsNegativeTestJSON-802831167-project-member] Lock "1954e745-783e-417e-aafc-265da55066cf" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 164.800s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.463155] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 07ec4d0679364ea0849e9b1f1ee39fa9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 821.479650] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 07ec4d0679364ea0849e9b1f1ee39fa9 [ 821.744096] env[64020]: DEBUG nova.scheduler.client.report [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 821.746612] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 261fa394567d4618b59e8d4d43079e2c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 821.759533] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 261fa394567d4618b59e8d4d43079e2c [ 821.897805] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407884, 'name': CreateVM_Task} progress is 99%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 821.941659] env[64020]: INFO nova.compute.manager [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] [instance: 5ccefb28-25a8-4876-a31c-58c7edfc9570] Took 1.07 seconds to deallocate network for instance. [ 821.943604] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 640105a57f0449aa8a3f498747a88fee in queue reply_57893177120949e6a93cb88e15cd42b4 [ 821.966645] env[64020]: DEBUG nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 821.968493] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 08d075bdcbde4f05b96124a41012693b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 822.002852] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 640105a57f0449aa8a3f498747a88fee [ 822.031046] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08d075bdcbde4f05b96124a41012693b [ 822.249406] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.526s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 822.250189] env[64020]: DEBUG nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 822.251699] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg a3847dfb5ea8411aa7b9f66caebb0df1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 822.253112] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.816s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 822.254825] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg f06586dc37684e64bd60e3cacfd854ec in queue reply_57893177120949e6a93cb88e15cd42b4 [ 822.301735] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f06586dc37684e64bd60e3cacfd854ec [ 822.305920] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a3847dfb5ea8411aa7b9f66caebb0df1 [ 822.396658] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407884, 'name': CreateVM_Task, 'duration_secs': 0.57233} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 822.396838] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 822.397246] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 822.397688] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 822.397757] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 822.397970] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-031b1495-47ae-4705-85d2-d8254ad8d212 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.402548] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 822.402548] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52e46a96-17c8-89ba-3691-b339ca7d2ae9" [ 822.402548] env[64020]: _type = "Task" [ 822.402548] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 822.411093] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52e46a96-17c8-89ba-3691-b339ca7d2ae9, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 822.448999] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg 83e786f700f043b2a2b98fd0967f4dcd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 822.485803] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 83e786f700f043b2a2b98fd0967f4dcd [ 822.489244] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 822.760036] env[64020]: DEBUG nova.compute.utils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 822.760036] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 59349e5dde314003a1e0558a103ffc45 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 822.762483] env[64020]: DEBUG nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 822.762657] env[64020]: DEBUG nova.network.neutron [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 822.777093] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59349e5dde314003a1e0558a103ffc45 [ 822.917588] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52e46a96-17c8-89ba-3691-b339ca7d2ae9, 'name': SearchDatastore_Task, 'duration_secs': 0.030369} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 822.918249] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 822.918646] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 822.919053] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 822.919468] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 822.919795] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 822.920377] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-b73b68ca-42c2-46d7-a7ed-a84c3f83e024 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.928924] env[64020]: DEBUG nova.policy [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd496b47ef9db47fa97323baf6bbd63ed', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f1fb1191ee044c4f921f5b7935f0109d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 822.933319] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 822.933718] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 822.934593] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c44bb2e5-75e0-4125-a5ec-a57aac183e05 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.950065] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 822.950065] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5288e9df-46f1-68c1-d7ce-ee754995ad56" [ 822.950065] env[64020]: _type = "Task" [ 822.950065] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 822.968855] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5288e9df-46f1-68c1-d7ce-ee754995ad56, 'name': SearchDatastore_Task, 'duration_secs': 0.008803} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 822.971801] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f3c956c7-cb60-44ca-a1be-d55187756140 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.982159] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 822.982159] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5210d965-60bb-6e2d-200f-55879fcf2b50" [ 822.982159] env[64020]: _type = "Task" [ 822.982159] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 822.993017] env[64020]: INFO nova.scheduler.client.report [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Deleted allocations for instance 5ccefb28-25a8-4876-a31c-58c7edfc9570 [ 823.006172] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Expecting reply to msg bb4e71f25889423ebaed2f27d976e02d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 823.010279] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5210d965-60bb-6e2d-200f-55879fcf2b50, 'name': SearchDatastore_Task, 'duration_secs': 0.009121} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 823.011298] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 823.011298] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] e27f988a-a64b-407b-ad66-4bed7ee45b52/e27f988a-a64b-407b-ad66-4bed7ee45b52.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 823.013873] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-d6d1377e-e5e8-4aad-a758-4378a291adaa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.021347] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb4e71f25889423ebaed2f27d976e02d [ 823.051121] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 823.051121] env[64020]: value = "task-407885" [ 823.051121] env[64020]: _type = "Task" [ 823.051121] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 823.060737] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407885, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 823.177951] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5876bd3d-0b5e-4103-85a1-f1104d9629eb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.186462] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ecff346f-918a-413f-9ec2-fa8643f36adf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.226029] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dbfa6abb-a9e4-42e3-aa31-413849279fc2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.234289] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1603de9-22bc-4c0d-85c4-3706b4a3949d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.249225] env[64020]: DEBUG nova.compute.provider_tree [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 823.249811] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 9f8fe4c37fc14a7281570c4eb650d2b0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 823.261789] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f8fe4c37fc14a7281570c4eb650d2b0 [ 823.263715] env[64020]: DEBUG nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 823.265881] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg fde0263c642144438191c0a47e0940a7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 823.316110] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fde0263c642144438191c0a47e0940a7 [ 823.511580] env[64020]: DEBUG oslo_concurrency.lockutils [None req-89d00a43-bb4e-4869-98fb-54f694b686ea tempest-ServersTestBootFromVolume-1164671616 tempest-ServersTestBootFromVolume-1164671616-project-member] Lock "5ccefb28-25a8-4876-a31c-58c7edfc9570" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 158.400s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 823.512040] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 8bef72d1477745f69bda2ebc6a7f9bc9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 823.525963] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8bef72d1477745f69bda2ebc6a7f9bc9 [ 823.562359] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407885, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.509783} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 823.563186] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] e27f988a-a64b-407b-ad66-4bed7ee45b52/e27f988a-a64b-407b-ad66-4bed7ee45b52.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 823.563655] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 823.564058] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-efb5d774-8eac-4455-96d8-601a5d605839 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.570169] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 823.570169] env[64020]: value = "task-407887" [ 823.570169] env[64020]: _type = "Task" [ 823.570169] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 823.581029] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407887, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 823.754087] env[64020]: DEBUG nova.scheduler.client.report [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 823.756566] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 0b0e9d75d7d74b11bcbfce6b188cdad2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 823.769426] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b0e9d75d7d74b11bcbfce6b188cdad2 [ 823.782556] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 999e9b4aa4c1456da67db519ddffd1b6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 823.822178] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 999e9b4aa4c1456da67db519ddffd1b6 [ 824.018619] env[64020]: DEBUG nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 824.024026] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 341f7a93af6d427d884d458a2fdd7826 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 824.024026] env[64020]: DEBUG nova.network.neutron [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Successfully created port: 112e5229-7ce7-4a48-8233-fa58ce776b1a {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 824.084380] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 341f7a93af6d427d884d458a2fdd7826 [ 824.084761] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407887, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.074143} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 824.085003] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 824.085795] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9b75b97-73bf-46e0-a308-4b0fc7275748 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.119227] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Reconfiguring VM instance instance-00000031 to attach disk [datastore1] e27f988a-a64b-407b-ad66-4bed7ee45b52/e27f988a-a64b-407b-ad66-4bed7ee45b52.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 824.120562] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-7625cf0a-c0f8-4adc-974e-4def33c1f913 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.142599] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 824.142599] env[64020]: value = "task-407888" [ 824.142599] env[64020]: _type = "Task" [ 824.142599] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 824.151194] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407888, 'name': ReconfigVM_Task} progress is 6%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 824.258996] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.006s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 824.259617] env[64020]: ERROR nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e13ceee3-08b5-4500-b589-817077fa9ddb, please check neutron logs for more information. [ 824.259617] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Traceback (most recent call last): [ 824.259617] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 824.259617] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] self.driver.spawn(context, instance, image_meta, [ 824.259617] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 824.259617] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 824.259617] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 824.259617] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] vm_ref = self.build_virtual_machine(instance, [ 824.259617] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 824.259617] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] vif_infos = vmwarevif.get_vif_info(self._session, [ 824.259617] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] for vif in network_info: [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] return self._sync_wrapper(fn, *args, **kwargs) [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] self.wait() [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] self[:] = self._gt.wait() [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] return self._exit_event.wait() [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] current.throw(*self._exc) [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 824.260043] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] result = function(*args, **kwargs) [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] return func(*args, **kwargs) [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] raise e [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] nwinfo = self.network_api.allocate_for_instance( [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] created_port_ids = self._update_ports_for_instance( [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] with excutils.save_and_reraise_exception(): [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] self.force_reraise() [ 824.260474] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 824.260902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] raise self.value [ 824.260902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 824.260902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] updated_port = self._update_port( [ 824.260902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 824.260902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] _ensure_no_port_binding_failure(port) [ 824.260902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 824.260902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] raise exception.PortBindingFailed(port_id=port['id']) [ 824.260902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] nova.exception.PortBindingFailed: Binding failed for port e13ceee3-08b5-4500-b589-817077fa9ddb, please check neutron logs for more information. [ 824.260902] env[64020]: ERROR nova.compute.manager [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] [ 824.260902] env[64020]: DEBUG nova.compute.utils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Binding failed for port e13ceee3-08b5-4500-b589-817077fa9ddb, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 824.262049] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.962s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 824.263523] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 379604829f944c82a45cb25dde3f2257 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 824.265908] env[64020]: DEBUG nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Build of instance a39b40c8-0340-4778-8b43-f29ed51664bc was re-scheduled: Binding failed for port e13ceee3-08b5-4500-b589-817077fa9ddb, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 824.266256] env[64020]: DEBUG nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 824.266509] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Acquiring lock "refresh_cache-a39b40c8-0340-4778-8b43-f29ed51664bc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 824.266661] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Acquired lock "refresh_cache-a39b40c8-0340-4778-8b43-f29ed51664bc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 824.266821] env[64020]: DEBUG nova.network.neutron [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 824.267208] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg f5b8e492faf145ecaa5fccbaf251f8de in queue reply_57893177120949e6a93cb88e15cd42b4 [ 824.285208] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5b8e492faf145ecaa5fccbaf251f8de [ 824.286443] env[64020]: DEBUG nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 824.325359] env[64020]: DEBUG nova.virt.hardware [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 824.325757] env[64020]: DEBUG nova.virt.hardware [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 824.325855] env[64020]: DEBUG nova.virt.hardware [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 824.326030] env[64020]: DEBUG nova.virt.hardware [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 824.326175] env[64020]: DEBUG nova.virt.hardware [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 824.326321] env[64020]: DEBUG nova.virt.hardware [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 824.326529] env[64020]: DEBUG nova.virt.hardware [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 824.326687] env[64020]: DEBUG nova.virt.hardware [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 824.326848] env[64020]: DEBUG nova.virt.hardware [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 824.327001] env[64020]: DEBUG nova.virt.hardware [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 824.327167] env[64020]: DEBUG nova.virt.hardware [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 824.328105] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2e5f5ca-c4c3-4d25-bad6-a4616975255d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.333980] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 379604829f944c82a45cb25dde3f2257 [ 824.338906] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44c580a8-f1f9-4ac7-b6ba-3f5a4ed988eb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.549210] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 824.652642] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407888, 'name': ReconfigVM_Task, 'duration_secs': 0.262958} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 824.652993] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Reconfigured VM instance instance-00000031 to attach disk [datastore1] e27f988a-a64b-407b-ad66-4bed7ee45b52/e27f988a-a64b-407b-ad66-4bed7ee45b52.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 824.653515] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-1cb212fb-c89e-4227-9459-77af228a06c9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.662360] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 824.662360] env[64020]: value = "task-407889" [ 824.662360] env[64020]: _type = "Task" [ 824.662360] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 824.675198] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407889, 'name': Rename_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 824.808517] env[64020]: DEBUG nova.network.neutron [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 824.962031] env[64020]: DEBUG nova.network.neutron [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 824.962604] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 7c35c2f04e864d66aa3dbb9e29efd84c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 824.975289] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c35c2f04e864d66aa3dbb9e29efd84c [ 825.074024] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Acquiring lock "a5567127-3d68-4c37-a8aa-03c5579cc18c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 825.074311] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Lock "a5567127-3d68-4c37-a8aa-03c5579cc18c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 825.172622] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407889, 'name': Rename_Task, 'duration_secs': 0.12818} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 825.172983] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 825.173299] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-e8605f8f-59db-4a37-8faf-ff3aaa0ca7b8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.177152] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16f7cca7-5ab4-4f73-a9fa-0e3efc19c410 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.189360] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99d7c372-dfa0-49ec-8714-aa49e94f485d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.193058] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 825.193058] env[64020]: value = "task-407890" [ 825.193058] env[64020]: _type = "Task" [ 825.193058] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 825.233543] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c44afc5-225c-4171-97a4-7487711759e3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.239998] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407890, 'name': PowerOnVM_Task} progress is 66%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 825.246793] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02a62137-ed02-4c4d-9b3b-c664d5091e51 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.265469] env[64020]: DEBUG nova.compute.provider_tree [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 825.266013] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg bf2f983b34224965b652cd90945231b5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 825.275723] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf2f983b34224965b652cd90945231b5 [ 825.465737] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Releasing lock "refresh_cache-a39b40c8-0340-4778-8b43-f29ed51664bc" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 825.465945] env[64020]: DEBUG nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 825.466108] env[64020]: DEBUG nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 825.466270] env[64020]: DEBUG nova.network.neutron [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 825.490659] env[64020]: DEBUG nova.network.neutron [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 825.491266] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 91a1cc03a6944ecd86605e52ed0dccd6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 825.499029] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91a1cc03a6944ecd86605e52ed0dccd6 [ 825.703026] env[64020]: DEBUG oslo_vmware.api [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407890, 'name': PowerOnVM_Task, 'duration_secs': 0.423544} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 825.703288] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 825.703500] env[64020]: DEBUG nova.compute.manager [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 825.704289] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc7aeedb-a126-420b-9d04-6b6b23a6ad1a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.711500] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 555bb4f84bbc46f9a69734bea3b64d0e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 825.763821] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 555bb4f84bbc46f9a69734bea3b64d0e [ 825.770035] env[64020]: DEBUG nova.scheduler.client.report [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 825.772564] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 32c13a31b3d54a7cab2134f78d1c58f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 825.786401] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 32c13a31b3d54a7cab2134f78d1c58f5 [ 825.998752] env[64020]: DEBUG nova.network.neutron [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 825.998752] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 6383834b497b42d3bdd28dba5c48d1d8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 826.007821] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6383834b497b42d3bdd28dba5c48d1d8 [ 826.222249] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.274732] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.013s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.275399] env[64020]: ERROR nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port bcd04248-654c-44de-ab87-7f1272bc06d3, please check neutron logs for more information. [ 826.275399] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Traceback (most recent call last): [ 826.275399] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 826.275399] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] self.driver.spawn(context, instance, image_meta, [ 826.275399] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 826.275399] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 826.275399] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 826.275399] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] vm_ref = self.build_virtual_machine(instance, [ 826.275399] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 826.275399] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] vif_infos = vmwarevif.get_vif_info(self._session, [ 826.275399] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] for vif in network_info: [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] return self._sync_wrapper(fn, *args, **kwargs) [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] self.wait() [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] self[:] = self._gt.wait() [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] return self._exit_event.wait() [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] current.throw(*self._exc) [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 826.275903] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] result = function(*args, **kwargs) [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] return func(*args, **kwargs) [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] raise e [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] nwinfo = self.network_api.allocate_for_instance( [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] created_port_ids = self._update_ports_for_instance( [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] with excutils.save_and_reraise_exception(): [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] self.force_reraise() [ 826.276428] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 826.276837] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] raise self.value [ 826.276837] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 826.276837] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] updated_port = self._update_port( [ 826.276837] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 826.276837] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] _ensure_no_port_binding_failure(port) [ 826.276837] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 826.276837] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] raise exception.PortBindingFailed(port_id=port['id']) [ 826.276837] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] nova.exception.PortBindingFailed: Binding failed for port bcd04248-654c-44de-ab87-7f1272bc06d3, please check neutron logs for more information. [ 826.276837] env[64020]: ERROR nova.compute.manager [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] [ 826.276837] env[64020]: DEBUG nova.compute.utils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Binding failed for port bcd04248-654c-44de-ab87-7f1272bc06d3, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 826.277459] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.501s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.279187] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 1ad63abc6fe6455e9036308d4cf9ac78 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 826.280399] env[64020]: DEBUG nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Build of instance 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c was re-scheduled: Binding failed for port bcd04248-654c-44de-ab87-7f1272bc06d3, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 826.280813] env[64020]: DEBUG nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 826.281032] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Acquiring lock "refresh_cache-73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 826.281206] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Acquired lock "refresh_cache-73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 826.281706] env[64020]: DEBUG nova.network.neutron [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 826.282114] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 77fd3e36fc56466e8c28cc26a436df62 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 826.299538] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 77fd3e36fc56466e8c28cc26a436df62 [ 826.312773] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 81605ee359e249959da3921cb732de79 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 826.339368] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ad63abc6fe6455e9036308d4cf9ac78 [ 826.341060] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81605ee359e249959da3921cb732de79 [ 826.499931] env[64020]: INFO nova.compute.manager [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] [instance: a39b40c8-0340-4778-8b43-f29ed51664bc] Took 1.03 seconds to deallocate network for instance. [ 826.501719] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 5bd49c6c8d564770973e4d32c8824977 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 826.537303] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5bd49c6c8d564770973e4d32c8824977 [ 826.815321] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquiring lock "e27f988a-a64b-407b-ad66-4bed7ee45b52" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.815597] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "e27f988a-a64b-407b-ad66-4bed7ee45b52" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.815820] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquiring lock "e27f988a-a64b-407b-ad66-4bed7ee45b52-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.816030] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "e27f988a-a64b-407b-ad66-4bed7ee45b52-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.816207] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "e27f988a-a64b-407b-ad66-4bed7ee45b52-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.818921] env[64020]: INFO nova.compute.manager [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Terminating instance [ 826.828462] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquiring lock "refresh_cache-e27f988a-a64b-407b-ad66-4bed7ee45b52" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 826.828462] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquired lock "refresh_cache-e27f988a-a64b-407b-ad66-4bed7ee45b52" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 826.828462] env[64020]: DEBUG nova.network.neutron [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 826.828462] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg e1e1c7191b064a72ab6ce413315a6d68 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 826.844348] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e1e1c7191b064a72ab6ce413315a6d68 [ 826.845634] env[64020]: DEBUG nova.network.neutron [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 826.878396] env[64020]: DEBUG nova.compute.manager [req-3df6ba5b-296b-4bcf-8684-8d8a6c3c5359 req-3f891e41-35a5-4919-beff-585676be3b27 service nova] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Received event network-changed-112e5229-7ce7-4a48-8233-fa58ce776b1a {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 826.878588] env[64020]: DEBUG nova.compute.manager [req-3df6ba5b-296b-4bcf-8684-8d8a6c3c5359 req-3f891e41-35a5-4919-beff-585676be3b27 service nova] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Refreshing instance network info cache due to event network-changed-112e5229-7ce7-4a48-8233-fa58ce776b1a. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 826.878778] env[64020]: DEBUG oslo_concurrency.lockutils [req-3df6ba5b-296b-4bcf-8684-8d8a6c3c5359 req-3f891e41-35a5-4919-beff-585676be3b27 service nova] Acquiring lock "refresh_cache-9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 826.878980] env[64020]: DEBUG oslo_concurrency.lockutils [req-3df6ba5b-296b-4bcf-8684-8d8a6c3c5359 req-3f891e41-35a5-4919-beff-585676be3b27 service nova] Acquired lock "refresh_cache-9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 826.879103] env[64020]: DEBUG nova.network.neutron [req-3df6ba5b-296b-4bcf-8684-8d8a6c3c5359 req-3f891e41-35a5-4919-beff-585676be3b27 service nova] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Refreshing network info cache for port 112e5229-7ce7-4a48-8233-fa58ce776b1a {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 826.879478] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3df6ba5b-296b-4bcf-8684-8d8a6c3c5359 req-3f891e41-35a5-4919-beff-585676be3b27 service nova] Expecting reply to msg e78d18c23bd74d96a3317cab34b620b5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 826.887837] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e78d18c23bd74d96a3317cab34b620b5 [ 826.970030] env[64020]: ERROR nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 112e5229-7ce7-4a48-8233-fa58ce776b1a, please check neutron logs for more information. [ 826.970030] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 826.970030] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 826.970030] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 826.970030] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 826.970030] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 826.970030] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 826.970030] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 826.970030] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 826.970030] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 826.970030] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 826.970030] env[64020]: ERROR nova.compute.manager raise self.value [ 826.970030] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 826.970030] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 826.970030] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 826.970030] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 826.970868] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 826.970868] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 826.970868] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 112e5229-7ce7-4a48-8233-fa58ce776b1a, please check neutron logs for more information. [ 826.970868] env[64020]: ERROR nova.compute.manager [ 826.970868] env[64020]: Traceback (most recent call last): [ 826.970868] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 826.970868] env[64020]: listener.cb(fileno) [ 826.970868] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 826.970868] env[64020]: result = function(*args, **kwargs) [ 826.970868] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 826.970868] env[64020]: return func(*args, **kwargs) [ 826.970868] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 826.970868] env[64020]: raise e [ 826.970868] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 826.970868] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 826.970868] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 826.970868] env[64020]: created_port_ids = self._update_ports_for_instance( [ 826.970868] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 826.970868] env[64020]: with excutils.save_and_reraise_exception(): [ 826.970868] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 826.970868] env[64020]: self.force_reraise() [ 826.970868] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 826.970868] env[64020]: raise self.value [ 826.970868] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 826.970868] env[64020]: updated_port = self._update_port( [ 826.970868] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 826.970868] env[64020]: _ensure_no_port_binding_failure(port) [ 826.970868] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 826.970868] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 826.971898] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 112e5229-7ce7-4a48-8233-fa58ce776b1a, please check neutron logs for more information. [ 826.971898] env[64020]: Removing descriptor: 18 [ 826.971898] env[64020]: ERROR nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 112e5229-7ce7-4a48-8233-fa58ce776b1a, please check neutron logs for more information. [ 826.971898] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Traceback (most recent call last): [ 826.971898] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 826.971898] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] yield resources [ 826.971898] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 826.971898] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] self.driver.spawn(context, instance, image_meta, [ 826.971898] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 826.971898] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 826.971898] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 826.971898] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] vm_ref = self.build_virtual_machine(instance, [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] vif_infos = vmwarevif.get_vif_info(self._session, [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] for vif in network_info: [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] return self._sync_wrapper(fn, *args, **kwargs) [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] self.wait() [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] self[:] = self._gt.wait() [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] return self._exit_event.wait() [ 826.973396] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] result = hub.switch() [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] return self.greenlet.switch() [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] result = function(*args, **kwargs) [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] return func(*args, **kwargs) [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] raise e [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] nwinfo = self.network_api.allocate_for_instance( [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 826.973844] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] created_port_ids = self._update_ports_for_instance( [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] with excutils.save_and_reraise_exception(): [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] self.force_reraise() [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] raise self.value [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] updated_port = self._update_port( [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] _ensure_no_port_binding_failure(port) [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 826.974473] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] raise exception.PortBindingFailed(port_id=port['id']) [ 826.975319] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] nova.exception.PortBindingFailed: Binding failed for port 112e5229-7ce7-4a48-8233-fa58ce776b1a, please check neutron logs for more information. [ 826.975319] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] [ 826.975319] env[64020]: INFO nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Terminating instance [ 826.975319] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 827.006060] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 8f446e4766944e88a1876633d98b6112 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 827.054991] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f446e4766944e88a1876633d98b6112 [ 827.231095] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25f07c97-8ec3-412a-ad99-906bbe625be8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.231095] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bcfae002-cf02-4eeb-b0b5-2db7abe2822e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.252814] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b314dfe5-e81f-4560-86f2-93268e05e7f0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.263948] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3baeed7-63df-4312-9118-f99a66f0027a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.277422] env[64020]: DEBUG nova.compute.provider_tree [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 827.278087] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 1466980aa1104c2aa159ead85b1df8e1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 827.297656] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1466980aa1104c2aa159ead85b1df8e1 [ 827.306985] env[64020]: DEBUG nova.network.neutron [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 827.307507] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 2c09381180d34e529ffea0fa2bd2f2b0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 827.334781] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c09381180d34e529ffea0fa2bd2f2b0 [ 827.408767] env[64020]: DEBUG nova.network.neutron [req-3df6ba5b-296b-4bcf-8684-8d8a6c3c5359 req-3f891e41-35a5-4919-beff-585676be3b27 service nova] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 827.416031] env[64020]: DEBUG nova.network.neutron [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 827.533508] env[64020]: INFO nova.scheduler.client.report [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Deleted allocations for instance a39b40c8-0340-4778-8b43-f29ed51664bc [ 827.548911] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Expecting reply to msg 382d21f5949e498291b018aaa2ddae2b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 827.563684] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 382d21f5949e498291b018aaa2ddae2b [ 827.582533] env[64020]: DEBUG nova.network.neutron [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 827.582533] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg b6f5f4d15b4e4261b2fdf6a17b465e4a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 827.589434] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b6f5f4d15b4e4261b2fdf6a17b465e4a [ 827.592456] env[64020]: DEBUG nova.network.neutron [req-3df6ba5b-296b-4bcf-8684-8d8a6c3c5359 req-3f891e41-35a5-4919-beff-585676be3b27 service nova] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 827.593341] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3df6ba5b-296b-4bcf-8684-8d8a6c3c5359 req-3f891e41-35a5-4919-beff-585676be3b27 service nova] Expecting reply to msg 77b5b7f5082545c295ee1298ed3e0fdc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 827.600484] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 77b5b7f5082545c295ee1298ed3e0fdc [ 827.788195] env[64020]: DEBUG nova.scheduler.client.report [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 827.790828] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 374435ba1596425592806b808eab566e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 827.804702] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 374435ba1596425592806b808eab566e [ 827.809071] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Releasing lock "refresh_cache-73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 827.809290] env[64020]: DEBUG nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 827.809481] env[64020]: DEBUG nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 827.809659] env[64020]: DEBUG nova.network.neutron [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 827.834136] env[64020]: DEBUG nova.network.neutron [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 827.834713] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg d4e957fe72084556af236dfaad20ef43 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 827.841902] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d4e957fe72084556af236dfaad20ef43 [ 828.052723] env[64020]: DEBUG oslo_concurrency.lockutils [None req-112dc60c-d737-4cbf-81e8-167ecbc1d367 tempest-TenantUsagesTestJSON-639123061 tempest-TenantUsagesTestJSON-639123061-project-member] Lock "a39b40c8-0340-4778-8b43-f29ed51664bc" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 153.233s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 828.053359] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 7ba6482a43b14b309fc56dd41ee65146 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 828.075247] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ba6482a43b14b309fc56dd41ee65146 [ 828.086185] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Releasing lock "refresh_cache-e27f988a-a64b-407b-ad66-4bed7ee45b52" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 828.086185] env[64020]: DEBUG nova.compute.manager [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 828.086185] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 828.086185] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7eb58933-c7c9-4972-b458-b4031a98660b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.094268] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 828.094753] env[64020]: DEBUG oslo_concurrency.lockutils [req-3df6ba5b-296b-4bcf-8684-8d8a6c3c5359 req-3f891e41-35a5-4919-beff-585676be3b27 service nova] Releasing lock "refresh_cache-9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 828.095071] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-08278ac5-be7e-4d3b-8c05-46b0678fa504 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.097329] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 828.097329] env[64020]: DEBUG nova.network.neutron [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 828.103563] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 64b607c4633e46748d35a17393964ebc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 828.111395] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64b607c4633e46748d35a17393964ebc [ 828.112079] env[64020]: DEBUG oslo_vmware.api [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 828.112079] env[64020]: value = "task-407891" [ 828.112079] env[64020]: _type = "Task" [ 828.112079] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 828.126602] env[64020]: DEBUG oslo_vmware.api [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407891, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 828.294103] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.017s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 828.294748] env[64020]: ERROR nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e83e0437-9379-4de3-b9fd-44e3086389c7, please check neutron logs for more information. [ 828.294748] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Traceback (most recent call last): [ 828.294748] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 828.294748] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] self.driver.spawn(context, instance, image_meta, [ 828.294748] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 828.294748] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] self._vmops.spawn(context, instance, image_meta, injected_files, [ 828.294748] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 828.294748] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] vm_ref = self.build_virtual_machine(instance, [ 828.294748] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 828.294748] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] vif_infos = vmwarevif.get_vif_info(self._session, [ 828.294748] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] for vif in network_info: [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] return self._sync_wrapper(fn, *args, **kwargs) [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] self.wait() [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] self[:] = self._gt.wait() [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] return self._exit_event.wait() [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] current.throw(*self._exc) [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 828.295109] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] result = function(*args, **kwargs) [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] return func(*args, **kwargs) [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] raise e [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] nwinfo = self.network_api.allocate_for_instance( [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] created_port_ids = self._update_ports_for_instance( [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] with excutils.save_and_reraise_exception(): [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] self.force_reraise() [ 828.295466] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 828.295814] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] raise self.value [ 828.295814] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 828.295814] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] updated_port = self._update_port( [ 828.295814] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 828.295814] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] _ensure_no_port_binding_failure(port) [ 828.295814] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 828.295814] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] raise exception.PortBindingFailed(port_id=port['id']) [ 828.295814] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] nova.exception.PortBindingFailed: Binding failed for port e83e0437-9379-4de3-b9fd-44e3086389c7, please check neutron logs for more information. [ 828.295814] env[64020]: ERROR nova.compute.manager [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] [ 828.295814] env[64020]: DEBUG nova.compute.utils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Binding failed for port e83e0437-9379-4de3-b9fd-44e3086389c7, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 828.296866] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 14.403s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 828.297597] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 2ed352bce0cb482b94ceb3020cd81c2a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 828.298668] env[64020]: DEBUG nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Build of instance d6228174-ceb1-4ca0-b62b-27db15c20a85 was re-scheduled: Binding failed for port e83e0437-9379-4de3-b9fd-44e3086389c7, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 828.299092] env[64020]: DEBUG nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 828.299354] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquiring lock "refresh_cache-d6228174-ceb1-4ca0-b62b-27db15c20a85" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 828.299496] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquired lock "refresh_cache-d6228174-ceb1-4ca0-b62b-27db15c20a85" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 828.299652] env[64020]: DEBUG nova.network.neutron [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 828.300091] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg fb80433194c4418bb2ffa1a73be176eb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 828.306553] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb80433194c4418bb2ffa1a73be176eb [ 828.314138] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ed352bce0cb482b94ceb3020cd81c2a [ 828.337469] env[64020]: DEBUG nova.network.neutron [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 828.338012] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg ff3b52b6a93d449c864929e102150857 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 828.358109] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff3b52b6a93d449c864929e102150857 [ 828.556059] env[64020]: DEBUG nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 828.557862] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg a822e62b62974e7db2fe74b8de63cc53 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 828.602109] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a822e62b62974e7db2fe74b8de63cc53 [ 828.618498] env[64020]: DEBUG nova.network.neutron [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 828.625057] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquiring lock "3c5781a8-df02-4b4f-ad3f-c04db921748f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 828.625394] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lock "3c5781a8-df02-4b4f-ad3f-c04db921748f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 828.630086] env[64020]: DEBUG oslo_vmware.api [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407891, 'name': PowerOffVM_Task, 'duration_secs': 0.200686} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 828.630537] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 828.630537] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 828.630721] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-5e8131ea-86f3-43d4-acb9-4ddac867c396 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.655773] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 828.656238] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Deleting contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 828.656433] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Deleting the datastore file [datastore1] e27f988a-a64b-407b-ad66-4bed7ee45b52 {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 828.656754] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-a9d3ca8f-32a5-45d5-acd6-b80488a52f92 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.663274] env[64020]: DEBUG oslo_vmware.api [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for the task: (returnval){ [ 828.663274] env[64020]: value = "task-407893" [ 828.663274] env[64020]: _type = "Task" [ 828.663274] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 828.672911] env[64020]: DEBUG oslo_vmware.api [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407893, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 828.721349] env[64020]: DEBUG nova.network.neutron [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 828.721905] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg d1a2d622c1444597aac64f629fe3fd16 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 828.730616] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1a2d622c1444597aac64f629fe3fd16 [ 828.804874] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg e5e867999daa460a8f184bdac5c06c82 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 828.841056] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e5e867999daa460a8f184bdac5c06c82 [ 828.843193] env[64020]: INFO nova.compute.manager [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] [instance: 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c] Took 1.03 seconds to deallocate network for instance. [ 828.845692] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg 24d59facd3554b54b0b31313d6669a02 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 828.852080] env[64020]: DEBUG nova.network.neutron [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 828.893902] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 24d59facd3554b54b0b31313d6669a02 [ 828.959454] env[64020]: DEBUG nova.network.neutron [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 828.959979] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 2e72fa25984545fda0da029458959dee in queue reply_57893177120949e6a93cb88e15cd42b4 [ 828.971216] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2e72fa25984545fda0da029458959dee [ 829.079543] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 829.143027] env[64020]: DEBUG nova.compute.manager [req-1a1e992f-7c48-4c2a-86c7-7c8c5bd84aba req-643a54b7-ebba-4c4d-8440-4cace4ee6e80 service nova] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Received event network-vif-deleted-112e5229-7ce7-4a48-8233-fa58ce776b1a {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 829.172611] env[64020]: DEBUG oslo_vmware.api [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Task: {'id': task-407893, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.107185} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 829.173027] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 829.173409] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Deleted contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 829.173666] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 829.173858] env[64020]: INFO nova.compute.manager [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Took 1.09 seconds to destroy the instance on the hypervisor. [ 829.174131] env[64020]: DEBUG oslo.service.loopingcall [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 829.174281] env[64020]: DEBUG nova.compute.manager [-] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 829.174364] env[64020]: DEBUG nova.network.neutron [-] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 829.190581] env[64020]: DEBUG nova.network.neutron [-] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 829.191124] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 74ec7d6d24f84300825b4f8f20459f8f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 829.202757] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74ec7d6d24f84300825b4f8f20459f8f [ 829.234343] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 829.234343] env[64020]: DEBUG nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 829.234343] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 829.234343] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a03d6a37-c174-410f-9623-5a4d79a1a0ce {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 829.241772] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a788fac3-e4ed-4b86-b39c-3a86a4ee163a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 829.264507] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3 could not be found. [ 829.264819] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 829.265081] env[64020]: INFO nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Took 0.03 seconds to destroy the instance on the hypervisor. [ 829.265421] env[64020]: DEBUG oslo.service.loopingcall [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 829.265818] env[64020]: DEBUG nova.compute.manager [-] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 829.265818] env[64020]: DEBUG nova.network.neutron [-] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 829.280697] env[64020]: DEBUG nova.network.neutron [-] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 829.281388] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0491024756594cdb85b6d46b3ed91770 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 829.288273] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0491024756594cdb85b6d46b3ed91770 [ 829.326809] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 4bfdd9371f244ee28b2735d9dd83088b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 829.338372] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4bfdd9371f244ee28b2735d9dd83088b [ 829.357895] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg ca5a047d4df64b078b2a4103153f93d0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 829.395387] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca5a047d4df64b078b2a4103153f93d0 [ 829.465498] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Releasing lock "refresh_cache-d6228174-ceb1-4ca0-b62b-27db15c20a85" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 829.465498] env[64020]: DEBUG nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 829.465498] env[64020]: DEBUG nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 829.465498] env[64020]: DEBUG nova.network.neutron [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 829.479762] env[64020]: DEBUG nova.network.neutron [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 829.480400] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg de2c56de357b405b98796b0b5e70cd9d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 829.490759] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de2c56de357b405b98796b0b5e70cd9d [ 829.693611] env[64020]: DEBUG nova.network.neutron [-] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 829.694068] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ca29a497dbb44b6b8500f269e38cad46 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 829.703182] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca29a497dbb44b6b8500f269e38cad46 [ 829.783180] env[64020]: DEBUG nova.network.neutron [-] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 829.783641] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 107908c129af4f309f23c2a63bf709e2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 829.792645] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 107908c129af4f309f23c2a63bf709e2 [ 829.829621] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 829.830215] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 8989c3f89a284195b6828badfc69bd62 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 829.840844] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8989c3f89a284195b6828badfc69bd62 [ 829.883136] env[64020]: INFO nova.scheduler.client.report [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Deleted allocations for instance 73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c [ 829.896094] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Expecting reply to msg abcaf294d57448868b4db33970424a71 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 829.903800] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abcaf294d57448868b4db33970424a71 [ 829.985040] env[64020]: DEBUG nova.network.neutron [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 829.985606] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 7321447533ba4c7fb55c380633732c0a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 829.994850] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7321447533ba4c7fb55c380633732c0a [ 830.196268] env[64020]: INFO nova.compute.manager [-] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Took 1.02 seconds to deallocate network for instance. [ 830.200365] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 48f8320ca6a142d28366c92bf1f1177a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 830.230297] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 48f8320ca6a142d28366c92bf1f1177a [ 830.286536] env[64020]: INFO nova.compute.manager [-] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Took 1.02 seconds to deallocate network for instance. [ 830.288845] env[64020]: DEBUG nova.compute.claims [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 830.289016] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.332621] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance d6228174-ceb1-4ca0-b62b-27db15c20a85 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 830.332861] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance e27f988a-a64b-407b-ad66-4bed7ee45b52 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 830.332925] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 830.333499] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 3448b4c2448f4e7ab69e04d5ad0854a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 830.345701] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3448b4c2448f4e7ab69e04d5ad0854a2 [ 830.391448] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7bd78138-b322-4600-ba4f-163e3e9fd128 tempest-InstanceActionsTestJSON-1729325202 tempest-InstanceActionsTestJSON-1729325202-project-member] Lock "73f671f6-6ea0-4b79-a5f3-a22f3fc9f59c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 152.903s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 830.392223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 3386ee64852447e1b8604adc502b68d3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 830.410519] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3386ee64852447e1b8604adc502b68d3 [ 830.488790] env[64020]: INFO nova.compute.manager [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: d6228174-ceb1-4ca0-b62b-27db15c20a85] Took 1.03 seconds to deallocate network for instance. [ 830.490822] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 60a649740322405aabef5545ec9dfeea in queue reply_57893177120949e6a93cb88e15cd42b4 [ 830.535698] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60a649740322405aabef5545ec9dfeea [ 830.588292] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquiring lock "b1ec9141-54d5-4761-bd37-55a1fc998589" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.588524] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "b1ec9141-54d5-4761-bd37-55a1fc998589" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 830.708279] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.836068] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 995288b5-d79a-4af3-a1e8-3571fff2d356 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 830.836685] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg d806efe045cc4f7786b50a014e6be314 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 830.847654] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d806efe045cc4f7786b50a014e6be314 [ 830.904882] env[64020]: DEBUG nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 830.906599] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 244fe853b6b14aec8d7260f3a9be120d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 830.961717] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 244fe853b6b14aec8d7260f3a9be120d [ 830.997005] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg abec15d34b824694b11f61ed6101f05c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 831.053593] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abec15d34b824694b11f61ed6101f05c [ 831.078108] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquiring lock "5554874d-598f-4276-8778-1da6773be649" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.078339] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "5554874d-598f-4276-8778-1da6773be649" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.339917] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 45167852-b7c0-4614-89f1-f8f7fc2078f2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 831.339917] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 61efa1daf5894ed8ab674a64b8b4a228 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 831.350479] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 61efa1daf5894ed8ab674a64b8b4a228 [ 831.427418] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.521450] env[64020]: INFO nova.scheduler.client.report [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Deleted allocations for instance d6228174-ceb1-4ca0-b62b-27db15c20a85 [ 831.528761] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 72cd20f73db94ae2af80f39d2b728c3a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 831.543455] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72cd20f73db94ae2af80f39d2b728c3a [ 831.841929] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance d567d3dd-5820-4d6c-8922-3e39a1d9ebd6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 831.842576] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 8c8445ed1ba04f90a2bb749fd85833aa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 831.853732] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8c8445ed1ba04f90a2bb749fd85833aa [ 832.032468] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1e1b5918-f49b-4fad-bd41-27b07d7c4aed tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "d6228174-ceb1-4ca0-b62b-27db15c20a85" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 150.027s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 832.033033] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 63c13304aa5242b8bec8391e6828f500 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 832.046621] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 63c13304aa5242b8bec8391e6828f500 [ 832.345867] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance ecb7f281-1206-4693-845b-c59f639b0789 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 832.346448] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 5f5bd68f701a4b7e9921bf41b41922a0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 832.356594] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5f5bd68f701a4b7e9921bf41b41922a0 [ 832.535312] env[64020]: DEBUG nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 832.537111] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 93dc1e73cffd4a85bade9b88012f1903 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 832.584504] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93dc1e73cffd4a85bade9b88012f1903 [ 832.848624] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 8de44ad9-adb7-4589-b2d8-a2aeeb89892f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 832.849192] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg aca3f12d09c646869e7bbcc2c9d4877b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 832.858905] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aca3f12d09c646869e7bbcc2c9d4877b [ 833.058490] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.353716] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance f7f8d239-aff8-4cd7-afdd-974c53b3e563 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 833.353716] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 2d28829ae0c84956bf1580eadaec2110 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 833.364193] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d28829ae0c84956bf1580eadaec2110 [ 833.856076] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance dcb3a0b0-a558-45be-8208-038e465dc802 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 833.856712] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg be7b8449df0f4f0ca70b6109761ef687 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 833.867607] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be7b8449df0f4f0ca70b6109761ef687 [ 834.066726] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "77801ae9-89db-4dc7-af03-0646af73b121" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 834.066947] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "77801ae9-89db-4dc7-af03-0646af73b121" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 834.359327] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance c9a9eccd-4730-4815-8270-8272ac8bdec6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 834.359915] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 5e0b1fd32e174f2886f0851f4be522a5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 834.372305] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e0b1fd32e174f2886f0851f4be522a5 [ 834.575200] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquiring lock "186cd540-6bb2-407f-9adb-9ed841656713" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 834.575976] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "186cd540-6bb2-407f-9adb-9ed841656713" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 834.766853] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "15815ad0-f79a-4031-b625-22c5f93dc2c1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 834.767078] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "15815ad0-f79a-4031-b625-22c5f93dc2c1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 834.865246] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance f4682b82-7475-412b-9319-bee1f05a9c63 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 834.865878] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 8dbcee5ef427481ab9dbd8a95fbd6e70 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 834.878589] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8dbcee5ef427481ab9dbd8a95fbd6e70 [ 835.368923] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 1235c526-3cfb-42b0-8e2d-64cfc59ec35e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 835.369492] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 907758a159024967bb37427f642b9c16 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 835.384817] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 907758a159024967bb37427f642b9c16 [ 835.877050] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 8c4773b7-a8de-4de3-a91f-2252cc34d11b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 835.877694] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg ee4c7b1b3cee47c3b1f4b9580d33e438 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 835.889766] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquiring lock "51b7673b-15ff-4940-9336-b17bc712f0e3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 835.889981] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "51b7673b-15ff-4940-9336-b17bc712f0e3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 835.894177] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ee4c7b1b3cee47c3b1f4b9580d33e438 [ 836.382304] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 7ba3e477-a789-4beb-90ac-651520836a5d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 836.382304] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 555fcf56b653432390853a0fb7569704 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 836.390913] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 555fcf56b653432390853a0fb7569704 [ 836.883036] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 836.883624] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 68a6268ed30b4dd7b318c2918249dd4d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 836.893295] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 68a6268ed30b4dd7b318c2918249dd4d [ 837.387293] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance f4017c56-63b3-420d-91ab-4565d82305fd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 837.387293] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 454f6bfc13494b63a5e11590e5de5a42 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 837.396839] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 454f6bfc13494b63a5e11590e5de5a42 [ 837.890106] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 5a021861-9784-431d-b717-c7b24fe8525c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 837.890690] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 2b3236a469f64b3b9adef70d36ded1d1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 837.900689] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b3236a469f64b3b9adef70d36ded1d1 [ 838.393075] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.393491] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 4b831a04e66948a282fe00f19a893754 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 838.404187] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4b831a04e66948a282fe00f19a893754 [ 838.895711] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance a5567127-3d68-4c37-a8aa-03c5579cc18c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.896290] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 09d0e8e7e2a942d99d22facaa5460c1d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 838.910817] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 09d0e8e7e2a942d99d22facaa5460c1d [ 839.403582] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 3c5781a8-df02-4b4f-ad3f-c04db921748f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 839.403891] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Total usable vcpus: 48, total allocated vcpus: 2 {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 839.403986] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=896MB phys_disk=200GB used_disk=2GB total_vcpus=48 used_vcpus=2 pci_stats=[] {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 839.708661] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0304f1b4-9e16-40fd-8e62-b750aaad02af {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.716379] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-093ee0c5-04c2-4ffb-b457-843ce4b6275f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.749102] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e83e2f7-6a5b-4bc5-9a9f-d5df8d927d7e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.755955] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9799fa84-67fe-4bcb-9fa8-315e33f1ecd3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.768606] env[64020]: DEBUG nova.compute.provider_tree [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 839.769191] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 9f097fa16108469f83bcc212b8397318 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 839.777141] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f097fa16108469f83bcc212b8397318 [ 840.271818] env[64020]: DEBUG nova.scheduler.client.report [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 840.274244] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 0a1b508726434c31abdb63a56e317dba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 840.286310] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a1b508726434c31abdb63a56e317dba [ 840.781156] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=64020) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 840.781156] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 12.480s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 840.781156] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 24.107s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 840.781156] env[64020]: INFO nova.compute.claims [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 840.781156] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg e1577f6443214f2581a8735adce4d427 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 840.788632] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 840.788632] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Cleaning up deleted instances {{(pid=64020) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11226}} [ 840.788632] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 2c831d00c476402482c2bf7ca0b96569 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 840.814390] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c831d00c476402482c2bf7ca0b96569 [ 840.824989] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e1577f6443214f2581a8735adce4d427 [ 841.288223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 3fb3d397086641d4888f4f648a4f1818 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 841.290239] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] There are 1 instances to clean {{(pid=64020) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11235}} [ 841.290481] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: b3440b55-2469-46fa-ac2c-3e207bf530ec] Instance has had 0 of 5 cleanup attempts {{(pid=64020) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11239}} [ 841.291448] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg d4b471ae96ac458d94893864640635f2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 841.296230] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3fb3d397086641d4888f4f648a4f1818 [ 841.345372] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d4b471ae96ac458d94893864640635f2 [ 841.793264] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 841.793517] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Cleaning up deleted instances with incomplete migration {{(pid=64020) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11264}} [ 841.793759] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg ba05cbe88b5245b2bd9cf193a7045811 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 841.804921] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba05cbe88b5245b2bd9cf193a7045811 [ 841.897297] env[64020]: DEBUG nova.scheduler.client.report [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Refreshing inventories for resource provider 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 841.918076] env[64020]: DEBUG nova.scheduler.client.report [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Updating ProviderTree inventory for provider 092888c9-2221-4dfc-9104-eeeb335c764f from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 841.918298] env[64020]: DEBUG nova.compute.provider_tree [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Updating inventory in ProviderTree for provider 092888c9-2221-4dfc-9104-eeeb335c764f with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 841.939364] env[64020]: DEBUG nova.scheduler.client.report [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Refreshing aggregate associations for resource provider 092888c9-2221-4dfc-9104-eeeb335c764f, aggregates: None {{(pid=64020) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 841.958676] env[64020]: DEBUG nova.scheduler.client.report [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Refreshing trait associations for resource provider 092888c9-2221-4dfc-9104-eeeb335c764f, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_NODE {{(pid=64020) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 842.258068] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aee6a6b6-cd68-4cf6-bd90-cd5da1916f9d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 842.265857] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15c7bc38-3447-4cf9-b50c-a8e7fdf01fb2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 842.296500] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 842.296836] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg f09b0b8136e74fdbaaf6cb84eb74f9a8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 842.298639] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18ce8c79-111c-4c8a-acb2-ed30c9cf0b04 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 842.305800] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f09b0b8136e74fdbaaf6cb84eb74f9a8 [ 842.307110] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fdcc2f68-128b-49d7-8bcf-a11f8a0be437 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 842.320825] env[64020]: DEBUG nova.compute.provider_tree [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 842.321322] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 3dcd7e4b0d35439d9fc523df3f6448c6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 842.329930] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3dcd7e4b0d35439d9fc523df3f6448c6 [ 842.824181] env[64020]: DEBUG nova.scheduler.client.report [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 842.826575] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 58a3c8a8633f4f769e681ec42b24653d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 842.841354] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 58a3c8a8633f4f769e681ec42b24653d [ 843.329621] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.552s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 843.330141] env[64020]: DEBUG nova.compute.manager [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 843.332123] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 75bf2b2ed4e443ae8cd479d8b051b0da in queue reply_57893177120949e6a93cb88e15cd42b4 [ 843.333076] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 24.773s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 843.334372] env[64020]: INFO nova.compute.claims [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 843.335792] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 455dee036d8b4a699ec39a452ce4c7a0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 843.365874] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 75bf2b2ed4e443ae8cd479d8b051b0da [ 843.367096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 455dee036d8b4a699ec39a452ce4c7a0 [ 843.839697] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg f179fdf11d6045eb9229722dbe87a777 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 843.841418] env[64020]: DEBUG nova.compute.utils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 843.841954] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 50c3d9d78ec94bb0bb5a58e6c5a48230 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 843.842723] env[64020]: DEBUG nova.compute.manager [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Not allocating networking since 'none' was specified. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 843.850256] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f179fdf11d6045eb9229722dbe87a777 [ 843.851411] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50c3d9d78ec94bb0bb5a58e6c5a48230 [ 844.347450] env[64020]: DEBUG nova.compute.manager [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 844.349119] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 516decbcaa3a4e4e9e752b498a86aedf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 844.378666] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 516decbcaa3a4e4e9e752b498a86aedf [ 844.698291] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72d0d8cd-0a9b-47ec-b077-cb20d93242c2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.707178] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccd5b466-d503-41a2-8636-9c6c5f27063f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.737239] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d6fac63-8638-43fb-a4e9-4685a68856cb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.745783] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36490b8d-76fa-4dab-826d-5eb3e9c12f80 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.764162] env[64020]: DEBUG nova.compute.provider_tree [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 844.764595] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 4e5610338b094103a61dfb01d0cdf615 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 844.772188] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e5610338b094103a61dfb01d0cdf615 [ 844.853892] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 6801da2ac0f24be49eb813e6a9854715 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 844.882571] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6801da2ac0f24be49eb813e6a9854715 [ 845.268043] env[64020]: DEBUG nova.scheduler.client.report [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 845.270757] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 0779f4581b19455ba206d56e9a877a3a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 845.281604] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0779f4581b19455ba206d56e9a877a3a [ 845.356696] env[64020]: DEBUG nova.compute.manager [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 845.382094] env[64020]: DEBUG nova.virt.hardware [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 845.382359] env[64020]: DEBUG nova.virt.hardware [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 845.382525] env[64020]: DEBUG nova.virt.hardware [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 845.382702] env[64020]: DEBUG nova.virt.hardware [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 845.382843] env[64020]: DEBUG nova.virt.hardware [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 845.382984] env[64020]: DEBUG nova.virt.hardware [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 845.383203] env[64020]: DEBUG nova.virt.hardware [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 845.383378] env[64020]: DEBUG nova.virt.hardware [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 845.383545] env[64020]: DEBUG nova.virt.hardware [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 845.383702] env[64020]: DEBUG nova.virt.hardware [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 845.383870] env[64020]: DEBUG nova.virt.hardware [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 845.384715] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21e6d2b0-372b-4e62-a5f8-dbf400836ba4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.392497] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75a4d77f-f31a-4cf8-ae8f-02ca9a034c26 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.406216] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 845.411642] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Creating folder: Project (1f4f55e2120d40f0bccc8fa698419bd5). Parent ref: group-v110249. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 845.412243] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-4664a529-3893-4169-a72c-911ad923f3af {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.422992] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Created folder: Project (1f4f55e2120d40f0bccc8fa698419bd5) in parent group-v110249. [ 845.423174] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Creating folder: Instances. Parent ref: group-v110267. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 845.423386] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-abb849b4-7106-4444-b3cd-313058c92f52 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.430752] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Created folder: Instances in parent group-v110267. [ 845.430972] env[64020]: DEBUG oslo.service.loopingcall [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 845.431144] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 845.431377] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-f63f2e29-613e-4f99-b44c-f3e37c0dbe94 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.447534] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 845.447534] env[64020]: value = "task-407896" [ 845.447534] env[64020]: _type = "Task" [ 845.447534] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 845.454635] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407896, 'name': CreateVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 845.773739] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.441s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 845.774268] env[64020]: DEBUG nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 845.775988] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 2b77001070134b4f9397b4d259a6a847 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 845.778520] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.420s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 845.779888] env[64020]: INFO nova.compute.claims [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 845.781641] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 09fa4f42b04b49678912be120cf87c74 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 845.806959] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b77001070134b4f9397b4d259a6a847 [ 845.814648] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 09fa4f42b04b49678912be120cf87c74 [ 845.956628] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407896, 'name': CreateVM_Task} progress is 99%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 846.281014] env[64020]: DEBUG nova.compute.utils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 846.281014] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 8de7d9059e9145f890aeba7b8b9051a4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 846.281857] env[64020]: DEBUG nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 846.282026] env[64020]: DEBUG nova.network.neutron [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 846.285390] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg c9a8ee0ea180459281a2154d3c8c915f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 846.292172] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8de7d9059e9145f890aeba7b8b9051a4 [ 846.292664] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c9a8ee0ea180459281a2154d3c8c915f [ 846.334641] env[64020]: DEBUG nova.policy [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd9d8e5fd2147420d832f9fe736a49790', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f21b217d9fc14132b1fdb6dfa55493bd', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 846.457587] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407896, 'name': CreateVM_Task} progress is 99%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 846.583424] env[64020]: DEBUG nova.network.neutron [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Successfully created port: 3dce5495-91e7-461f-a6e6-b9be33af5bf0 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 846.796356] env[64020]: DEBUG nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 846.796356] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 797694a632d4499bb9a79ded32ab07cb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 846.839952] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 797694a632d4499bb9a79ded32ab07cb [ 846.958508] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407896, 'name': CreateVM_Task, 'duration_secs': 1.245151} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 846.958508] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 846.958824] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 846.958824] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 846.959217] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 846.959492] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-00349b74-b487-44a2-b4f8-f25a1d237cb1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 846.969329] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for the task: (returnval){ [ 846.969329] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]524736d3-f49c-696f-e897-d1002f5fd82c" [ 846.969329] env[64020]: _type = "Task" [ 846.969329] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 846.979914] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]524736d3-f49c-696f-e897-d1002f5fd82c, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 847.171898] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d824ca9a-c5d0-46b3-b43a-68b05046cf53 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.179451] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9161b362-691e-4adc-a1f8-223308cc61de {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.211525] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa3e4dcc-cf18-412c-975c-cf8ee0bc25a6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.219328] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48c14fb8-0bc5-41a2-a10f-a182b1459d8a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.233004] env[64020]: DEBUG nova.compute.provider_tree [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 847.233559] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg d070a9e9bada4c2fb91bb3c044c52ffb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 847.241335] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d070a9e9bada4c2fb91bb3c044c52ffb [ 847.308104] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg f78a690f373e401498a95a6f9f2d6f6b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 847.330171] env[64020]: DEBUG nova.compute.manager [req-2d0093a0-d2d2-4391-a284-6f8e7338ff0c req-33b0732d-5b26-42ad-a6e6-c3c3c35f05ed service nova] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Received event network-changed-3dce5495-91e7-461f-a6e6-b9be33af5bf0 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 847.330316] env[64020]: DEBUG nova.compute.manager [req-2d0093a0-d2d2-4391-a284-6f8e7338ff0c req-33b0732d-5b26-42ad-a6e6-c3c3c35f05ed service nova] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Refreshing instance network info cache due to event network-changed-3dce5495-91e7-461f-a6e6-b9be33af5bf0. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 847.330528] env[64020]: DEBUG oslo_concurrency.lockutils [req-2d0093a0-d2d2-4391-a284-6f8e7338ff0c req-33b0732d-5b26-42ad-a6e6-c3c3c35f05ed service nova] Acquiring lock "refresh_cache-45167852-b7c0-4614-89f1-f8f7fc2078f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 847.330663] env[64020]: DEBUG oslo_concurrency.lockutils [req-2d0093a0-d2d2-4391-a284-6f8e7338ff0c req-33b0732d-5b26-42ad-a6e6-c3c3c35f05ed service nova] Acquired lock "refresh_cache-45167852-b7c0-4614-89f1-f8f7fc2078f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 847.330815] env[64020]: DEBUG nova.network.neutron [req-2d0093a0-d2d2-4391-a284-6f8e7338ff0c req-33b0732d-5b26-42ad-a6e6-c3c3c35f05ed service nova] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Refreshing network info cache for port 3dce5495-91e7-461f-a6e6-b9be33af5bf0 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 847.331817] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-2d0093a0-d2d2-4391-a284-6f8e7338ff0c req-33b0732d-5b26-42ad-a6e6-c3c3c35f05ed service nova] Expecting reply to msg 1f71d5f17d384a958e72fe20680f8aaa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 847.336345] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f78a690f373e401498a95a6f9f2d6f6b [ 847.339227] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f71d5f17d384a958e72fe20680f8aaa [ 847.480228] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]524736d3-f49c-696f-e897-d1002f5fd82c, 'name': SearchDatastore_Task, 'duration_secs': 0.011865} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 847.480516] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 847.480738] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 847.480955] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 847.481090] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 847.481290] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 847.482330] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-85de00df-042d-4a2e-a21f-5b8e403f353a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.487546] env[64020]: ERROR nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3dce5495-91e7-461f-a6e6-b9be33af5bf0, please check neutron logs for more information. [ 847.487546] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 847.487546] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 847.487546] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 847.487546] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 847.487546] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 847.487546] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 847.487546] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 847.487546] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 847.487546] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 847.487546] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 847.487546] env[64020]: ERROR nova.compute.manager raise self.value [ 847.487546] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 847.487546] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 847.487546] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 847.487546] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 847.488093] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 847.488093] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 847.488093] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3dce5495-91e7-461f-a6e6-b9be33af5bf0, please check neutron logs for more information. [ 847.488093] env[64020]: ERROR nova.compute.manager [ 847.488421] env[64020]: Traceback (most recent call last): [ 847.488514] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 847.488514] env[64020]: listener.cb(fileno) [ 847.488594] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 847.488594] env[64020]: result = function(*args, **kwargs) [ 847.488664] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 847.488664] env[64020]: return func(*args, **kwargs) [ 847.488727] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 847.488727] env[64020]: raise e [ 847.488791] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 847.488791] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 847.488857] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 847.488857] env[64020]: created_port_ids = self._update_ports_for_instance( [ 847.488933] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 847.488933] env[64020]: with excutils.save_and_reraise_exception(): [ 847.488996] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 847.488996] env[64020]: self.force_reraise() [ 847.489058] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 847.489058] env[64020]: raise self.value [ 847.489124] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 847.489124] env[64020]: updated_port = self._update_port( [ 847.489203] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 847.489203] env[64020]: _ensure_no_port_binding_failure(port) [ 847.489261] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 847.489261] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 847.489327] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 3dce5495-91e7-461f-a6e6-b9be33af5bf0, please check neutron logs for more information. [ 847.489368] env[64020]: Removing descriptor: 18 [ 847.490577] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 847.490866] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 847.491697] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b9c8b2a1-076a-4723-8eba-66fdade80bd5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.496615] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for the task: (returnval){ [ 847.496615] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52f824bb-f5dd-7296-a282-e974f999132b" [ 847.496615] env[64020]: _type = "Task" [ 847.496615] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 847.504579] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52f824bb-f5dd-7296-a282-e974f999132b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 847.737520] env[64020]: DEBUG nova.scheduler.client.report [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 847.747190] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 1e553399054b4f4e94fe1f533bc964b6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 847.760519] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e553399054b4f4e94fe1f533bc964b6 [ 847.807579] env[64020]: DEBUG nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 847.834153] env[64020]: DEBUG nova.virt.hardware [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 847.834592] env[64020]: DEBUG nova.virt.hardware [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 847.834871] env[64020]: DEBUG nova.virt.hardware [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 847.835148] env[64020]: DEBUG nova.virt.hardware [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 847.835386] env[64020]: DEBUG nova.virt.hardware [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 847.835623] env[64020]: DEBUG nova.virt.hardware [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 847.835919] env[64020]: DEBUG nova.virt.hardware [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 847.836212] env[64020]: DEBUG nova.virt.hardware [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 847.836480] env[64020]: DEBUG nova.virt.hardware [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 847.836763] env[64020]: DEBUG nova.virt.hardware [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 847.837036] env[64020]: DEBUG nova.virt.hardware [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 847.839680] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7702186-8137-4692-b077-1632fab59989 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.850539] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3971b99-510a-4401-b428-a153d5aad05f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.864469] env[64020]: ERROR nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3dce5495-91e7-461f-a6e6-b9be33af5bf0, please check neutron logs for more information. [ 847.864469] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Traceback (most recent call last): [ 847.864469] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 847.864469] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] yield resources [ 847.864469] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 847.864469] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] self.driver.spawn(context, instance, image_meta, [ 847.864469] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 847.864469] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 847.864469] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 847.864469] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] vm_ref = self.build_virtual_machine(instance, [ 847.864469] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] vif_infos = vmwarevif.get_vif_info(self._session, [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] for vif in network_info: [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] return self._sync_wrapper(fn, *args, **kwargs) [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] self.wait() [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] self[:] = self._gt.wait() [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] return self._exit_event.wait() [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 847.864843] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] current.throw(*self._exc) [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] result = function(*args, **kwargs) [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] return func(*args, **kwargs) [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] raise e [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] nwinfo = self.network_api.allocate_for_instance( [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] created_port_ids = self._update_ports_for_instance( [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] with excutils.save_and_reraise_exception(): [ 847.865192] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 847.865525] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] self.force_reraise() [ 847.865525] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 847.865525] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] raise self.value [ 847.865525] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 847.865525] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] updated_port = self._update_port( [ 847.865525] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 847.865525] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] _ensure_no_port_binding_failure(port) [ 847.865525] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 847.865525] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] raise exception.PortBindingFailed(port_id=port['id']) [ 847.865525] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] nova.exception.PortBindingFailed: Binding failed for port 3dce5495-91e7-461f-a6e6-b9be33af5bf0, please check neutron logs for more information. [ 847.865525] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] [ 847.866095] env[64020]: INFO nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Terminating instance [ 847.868229] env[64020]: DEBUG nova.network.neutron [req-2d0093a0-d2d2-4391-a284-6f8e7338ff0c req-33b0732d-5b26-42ad-a6e6-c3c3c35f05ed service nova] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 847.870096] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Acquiring lock "refresh_cache-45167852-b7c0-4614-89f1-f8f7fc2078f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 847.937365] env[64020]: DEBUG nova.network.neutron [req-2d0093a0-d2d2-4391-a284-6f8e7338ff0c req-33b0732d-5b26-42ad-a6e6-c3c3c35f05ed service nova] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 847.938018] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-2d0093a0-d2d2-4391-a284-6f8e7338ff0c req-33b0732d-5b26-42ad-a6e6-c3c3c35f05ed service nova] Expecting reply to msg a01dac03b1974cb0b34f2e9bb503cb7a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 847.947309] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a01dac03b1974cb0b34f2e9bb503cb7a [ 848.007542] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52f824bb-f5dd-7296-a282-e974f999132b, 'name': SearchDatastore_Task, 'duration_secs': 0.007787} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 848.008656] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-ab376097-3d91-4001-b3f4-d034e7e90a7e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.013701] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for the task: (returnval){ [ 848.013701] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]529f73e9-1090-92f8-3ad6-969527803f67" [ 848.013701] env[64020]: _type = "Task" [ 848.013701] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 848.021397] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]529f73e9-1090-92f8-3ad6-969527803f67, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 848.250180] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.471s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 848.255681] env[64020]: DEBUG nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 848.255681] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 3b7d6f114f5749afa9276d33f78e4d29 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 848.255681] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 27.498s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 848.255681] env[64020]: INFO nova.compute.claims [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 848.257500] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 7d5d88acf85745fca71898b833c4b2fe in queue reply_57893177120949e6a93cb88e15cd42b4 [ 848.294113] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d5d88acf85745fca71898b833c4b2fe [ 848.294949] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b7d6f114f5749afa9276d33f78e4d29 [ 848.440443] env[64020]: DEBUG oslo_concurrency.lockutils [req-2d0093a0-d2d2-4391-a284-6f8e7338ff0c req-33b0732d-5b26-42ad-a6e6-c3c3c35f05ed service nova] Releasing lock "refresh_cache-45167852-b7c0-4614-89f1-f8f7fc2078f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 848.441001] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Acquired lock "refresh_cache-45167852-b7c0-4614-89f1-f8f7fc2078f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 848.441326] env[64020]: DEBUG nova.network.neutron [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 848.441974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 45b4996cfbcf41bf9fa3be6a43a13c77 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 848.449094] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 45b4996cfbcf41bf9fa3be6a43a13c77 [ 848.525296] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]529f73e9-1090-92f8-3ad6-969527803f67, 'name': SearchDatastore_Task, 'duration_secs': 0.008112} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 848.525808] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 848.526293] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 995288b5-d79a-4af3-a1e8-3571fff2d356/995288b5-d79a-4af3-a1e8-3571fff2d356.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 848.526718] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-7b416bf2-c744-4430-ac0b-eac87fafbc68 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.533136] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for the task: (returnval){ [ 848.533136] env[64020]: value = "task-407897" [ 848.533136] env[64020]: _type = "Task" [ 848.533136] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 848.541062] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407897, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 848.764636] env[64020]: DEBUG nova.compute.utils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 848.764636] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 1b77a7641fd14d4d9a1f6683f383e73a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 848.764636] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg e00de25459ae4ff9a9df0b7c1050b9b2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 848.765895] env[64020]: DEBUG nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 848.766214] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 848.777000] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b77a7641fd14d4d9a1f6683f383e73a [ 848.781346] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e00de25459ae4ff9a9df0b7c1050b9b2 [ 848.829892] env[64020]: DEBUG nova.policy [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '17c497749a024ed389ed988afd37327c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '74dfe9d51d8e4237b10602ad703c21f6', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 848.967564] env[64020]: DEBUG nova.network.neutron [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 849.045360] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407897, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.461186} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 849.045989] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 995288b5-d79a-4af3-a1e8-3571fff2d356/995288b5-d79a-4af3-a1e8-3571fff2d356.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 849.046375] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 849.046754] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-7ae445a0-91a5-4ae1-81de-e07a2bbfe9c1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.053031] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for the task: (returnval){ [ 849.053031] env[64020]: value = "task-407898" [ 849.053031] env[64020]: _type = "Task" [ 849.053031] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 849.061406] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407898, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 849.083594] env[64020]: DEBUG nova.network.neutron [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 849.084342] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 4990cf952a834118aa6dd9a8776d2973 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 849.092839] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4990cf952a834118aa6dd9a8776d2973 [ 849.124450] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Successfully created port: 6a31c3b4-3e9f-44ef-a054-d0e46cbcd727 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 849.266917] env[64020]: DEBUG nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 849.268850] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 06e10c5699fa4da18769b6fbf8258218 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 849.300043] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06e10c5699fa4da18769b6fbf8258218 [ 849.392775] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Successfully created port: 2917ab7f-495c-434a-af82-aae11c410b43 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 849.410804] env[64020]: DEBUG nova.compute.manager [req-2a3df917-45f1-420b-b501-420888328e1f req-83cf67e5-9558-4c3b-bfff-10876b677f4a service nova] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Received event network-vif-deleted-3dce5495-91e7-461f-a6e6-b9be33af5bf0 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 849.563069] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407898, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.058565} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 849.565390] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 849.566569] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-262ae679-49c6-46e7-aa89-ca73d955f3cc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.586231] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Reconfiguring VM instance instance-00000033 to attach disk [datastore1] 995288b5-d79a-4af3-a1e8-3571fff2d356/995288b5-d79a-4af3-a1e8-3571fff2d356.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 849.589121] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Releasing lock "refresh_cache-45167852-b7c0-4614-89f1-f8f7fc2078f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 849.589476] env[64020]: DEBUG nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 849.589659] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 849.589884] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-fbe22332-c811-42a1-94f7-e56752b0c184 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.603981] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0265e2e6-3df9-488f-bae7-5e1c15b548ed {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.613769] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84b7ab1d-2b40-4d3f-8f92-901be9f21191 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.624899] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for the task: (returnval){ [ 849.624899] env[64020]: value = "task-407899" [ 849.624899] env[64020]: _type = "Task" [ 849.624899] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 849.635591] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407899, 'name': ReconfigVM_Task} progress is 14%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 849.639786] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 45167852-b7c0-4614-89f1-f8f7fc2078f2 could not be found. [ 849.639994] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 849.640189] env[64020]: INFO nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Took 0.05 seconds to destroy the instance on the hypervisor. [ 849.640422] env[64020]: DEBUG oslo.service.loopingcall [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 849.642797] env[64020]: DEBUG nova.compute.manager [-] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 849.642890] env[64020]: DEBUG nova.network.neutron [-] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 849.646523] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Successfully created port: 09d8e2d5-8153-4202-bdb5-16f78cf27935 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 849.659812] env[64020]: DEBUG nova.network.neutron [-] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 849.660283] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8ccbf29560b24dc1837ad50310f8afb8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 849.669642] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ccbf29560b24dc1837ad50310f8afb8 [ 849.678824] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30f21239-60f8-4569-aa62-33e9506e5337 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.686423] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c43469ed-3a0c-4d56-bddc-69a217057c44 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.715990] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c2f7739-24e0-4e5f-a4b9-8111e5841917 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.723548] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c028dfe-5f76-46b2-9df8-d132c64aaa39 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.736508] env[64020]: DEBUG nova.compute.provider_tree [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 849.737143] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg c71bc682b1b34a45b95ddfdca52c2bc0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 849.744316] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c71bc682b1b34a45b95ddfdca52c2bc0 [ 849.777163] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg d3bb94a87d7049938766dc36805229f6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 849.806212] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d3bb94a87d7049938766dc36805229f6 [ 850.139550] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407899, 'name': ReconfigVM_Task, 'duration_secs': 0.258254} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 850.139812] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Reconfigured VM instance instance-00000033 to attach disk [datastore1] 995288b5-d79a-4af3-a1e8-3571fff2d356/995288b5-d79a-4af3-a1e8-3571fff2d356.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 850.140503] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-cf9719c0-b9a8-4978-824d-87e20ad35c83 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.149248] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for the task: (returnval){ [ 850.149248] env[64020]: value = "task-407900" [ 850.149248] env[64020]: _type = "Task" [ 850.149248] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 850.159673] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407900, 'name': Rename_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 850.162040] env[64020]: DEBUG nova.network.neutron [-] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 850.162478] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0f20459304f64c34823088054396958a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 850.172498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f20459304f64c34823088054396958a [ 850.239596] env[64020]: DEBUG nova.scheduler.client.report [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 850.242217] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 775beaf46ea04681ac306215d9084d68 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 850.254273] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 775beaf46ea04681ac306215d9084d68 [ 850.280475] env[64020]: DEBUG nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 850.305043] env[64020]: DEBUG nova.virt.hardware [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 850.305287] env[64020]: DEBUG nova.virt.hardware [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 850.305440] env[64020]: DEBUG nova.virt.hardware [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 850.305618] env[64020]: DEBUG nova.virt.hardware [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 850.305759] env[64020]: DEBUG nova.virt.hardware [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 850.305949] env[64020]: DEBUG nova.virt.hardware [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 850.306094] env[64020]: DEBUG nova.virt.hardware [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 850.306364] env[64020]: DEBUG nova.virt.hardware [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 850.306420] env[64020]: DEBUG nova.virt.hardware [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 850.306558] env[64020]: DEBUG nova.virt.hardware [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 850.306723] env[64020]: DEBUG nova.virt.hardware [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 850.307536] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1fa8ac6-c07e-4aa1-a929-6b8dcaac4662 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.315496] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60758ae0-626e-4ba3-98bc-dd481a3a39e8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.624662] env[64020]: ERROR nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6a31c3b4-3e9f-44ef-a054-d0e46cbcd727, please check neutron logs for more information. [ 850.624662] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 850.624662] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 850.624662] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 850.624662] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 850.624662] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 850.624662] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 850.624662] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 850.624662] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 850.624662] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 850.624662] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 850.624662] env[64020]: ERROR nova.compute.manager raise self.value [ 850.624662] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 850.624662] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 850.624662] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 850.624662] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 850.625036] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 850.625036] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 850.625036] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6a31c3b4-3e9f-44ef-a054-d0e46cbcd727, please check neutron logs for more information. [ 850.625036] env[64020]: ERROR nova.compute.manager [ 850.625036] env[64020]: Traceback (most recent call last): [ 850.625036] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 850.625036] env[64020]: listener.cb(fileno) [ 850.625036] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 850.625036] env[64020]: result = function(*args, **kwargs) [ 850.625036] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 850.625036] env[64020]: return func(*args, **kwargs) [ 850.625036] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 850.625036] env[64020]: raise e [ 850.625036] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 850.625036] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 850.625036] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 850.625036] env[64020]: created_port_ids = self._update_ports_for_instance( [ 850.625036] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 850.625036] env[64020]: with excutils.save_and_reraise_exception(): [ 850.625036] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 850.625036] env[64020]: self.force_reraise() [ 850.625036] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 850.625036] env[64020]: raise self.value [ 850.625036] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 850.625036] env[64020]: updated_port = self._update_port( [ 850.625036] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 850.625036] env[64020]: _ensure_no_port_binding_failure(port) [ 850.625036] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 850.625036] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 850.625659] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 6a31c3b4-3e9f-44ef-a054-d0e46cbcd727, please check neutron logs for more information. [ 850.625659] env[64020]: Removing descriptor: 18 [ 850.625659] env[64020]: ERROR nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6a31c3b4-3e9f-44ef-a054-d0e46cbcd727, please check neutron logs for more information. [ 850.625659] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Traceback (most recent call last): [ 850.625659] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 850.625659] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] yield resources [ 850.625659] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 850.625659] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] self.driver.spawn(context, instance, image_meta, [ 850.625659] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 850.625659] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 850.625659] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 850.625659] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] vm_ref = self.build_virtual_machine(instance, [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] vif_infos = vmwarevif.get_vif_info(self._session, [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] for vif in network_info: [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] return self._sync_wrapper(fn, *args, **kwargs) [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] self.wait() [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] self[:] = self._gt.wait() [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] return self._exit_event.wait() [ 850.625922] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] result = hub.switch() [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] return self.greenlet.switch() [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] result = function(*args, **kwargs) [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] return func(*args, **kwargs) [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] raise e [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] nwinfo = self.network_api.allocate_for_instance( [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 850.626486] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] created_port_ids = self._update_ports_for_instance( [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] with excutils.save_and_reraise_exception(): [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] self.force_reraise() [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] raise self.value [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] updated_port = self._update_port( [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] _ensure_no_port_binding_failure(port) [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 850.626760] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] raise exception.PortBindingFailed(port_id=port['id']) [ 850.627019] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] nova.exception.PortBindingFailed: Binding failed for port 6a31c3b4-3e9f-44ef-a054-d0e46cbcd727, please check neutron logs for more information. [ 850.627019] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] [ 850.627019] env[64020]: INFO nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Terminating instance [ 850.627701] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquiring lock "refresh_cache-d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 850.628558] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquired lock "refresh_cache-d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 850.628558] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 850.628558] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg b385365fc01d4b00826d770a9c6ce68b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 850.635122] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b385365fc01d4b00826d770a9c6ce68b [ 850.660024] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407900, 'name': Rename_Task, 'duration_secs': 0.130586} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 850.660024] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 850.660024] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-1c5c9d3b-d987-42ac-ad3a-d67fc192405e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.664680] env[64020]: INFO nova.compute.manager [-] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Took 1.02 seconds to deallocate network for instance. [ 850.667201] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for the task: (returnval){ [ 850.667201] env[64020]: value = "task-407901" [ 850.667201] env[64020]: _type = "Task" [ 850.667201] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 850.667699] env[64020]: DEBUG nova.compute.claims [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 850.667916] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 850.676043] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407901, 'name': PowerOnVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 850.750249] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.497s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 850.750860] env[64020]: DEBUG nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 850.752753] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 8acc957641ad4ab5bb91cf3319c552af in queue reply_57893177120949e6a93cb88e15cd42b4 [ 850.753820] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 28.265s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 850.755240] env[64020]: INFO nova.compute.claims [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 850.756771] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg b3816c93012443d0a216b2583fb221a3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 850.788842] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8acc957641ad4ab5bb91cf3319c552af [ 850.800031] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3816c93012443d0a216b2583fb221a3 [ 851.177656] env[64020]: DEBUG oslo_vmware.api [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407901, 'name': PowerOnVM_Task, 'duration_secs': 0.391718} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 851.178487] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 851.180234] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 851.180501] env[64020]: INFO nova.compute.manager [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Took 5.82 seconds to spawn the instance on the hypervisor. [ 851.180739] env[64020]: DEBUG nova.compute.manager [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 851.181777] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c72b0b3-2a69-4bea-bcb6-609733b703e9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.189278] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg bbd6e58deabe465fbe9aa34e62545f5b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 851.224032] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bbd6e58deabe465fbe9aa34e62545f5b [ 851.259420] env[64020]: DEBUG nova.compute.utils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 851.260072] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 2baaf3cbd9e749a9abb7c681330ff817 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 851.261678] env[64020]: DEBUG nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 851.261871] env[64020]: DEBUG nova.network.neutron [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 851.267665] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 5a7e917c7c954b0ba90eadfa5fd658a3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 851.271400] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2baaf3cbd9e749a9abb7c681330ff817 [ 851.275554] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a7e917c7c954b0ba90eadfa5fd658a3 [ 851.279904] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 851.280403] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 0ae06133e24f4509a24d9a1fc9512acd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 851.290382] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0ae06133e24f4509a24d9a1fc9512acd [ 851.318721] env[64020]: DEBUG nova.policy [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02b993f5bbd5442a8d2c06cff095e867', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e0c4cb7c0e9941f7a056fef708be822c', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 851.439826] env[64020]: DEBUG nova.compute.manager [req-dd2ca881-f580-4492-abe6-f6a81c22ff27 req-94d9af6b-344a-4dcf-819a-ecb0392b8255 service nova] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Received event network-changed-6a31c3b4-3e9f-44ef-a054-d0e46cbcd727 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 851.439826] env[64020]: DEBUG nova.compute.manager [req-dd2ca881-f580-4492-abe6-f6a81c22ff27 req-94d9af6b-344a-4dcf-819a-ecb0392b8255 service nova] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Refreshing instance network info cache due to event network-changed-6a31c3b4-3e9f-44ef-a054-d0e46cbcd727. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 851.439826] env[64020]: DEBUG oslo_concurrency.lockutils [req-dd2ca881-f580-4492-abe6-f6a81c22ff27 req-94d9af6b-344a-4dcf-819a-ecb0392b8255 service nova] Acquiring lock "refresh_cache-d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 851.698382] env[64020]: INFO nova.compute.manager [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Took 35.04 seconds to build instance. [ 851.698882] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 105e523d8cd3457e97f5be455d1cda1a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 851.713722] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 105e523d8cd3457e97f5be455d1cda1a [ 851.745847] env[64020]: DEBUG nova.network.neutron [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Successfully created port: 53782645-b8fb-4e78-bdaf-8f397474785d {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 851.762856] env[64020]: DEBUG nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 851.764901] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 6554ea65a73b41cb994faf7d243bb33a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 851.782912] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Releasing lock "refresh_cache-d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 851.783307] env[64020]: DEBUG nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 851.783542] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 851.783768] env[64020]: DEBUG oslo_concurrency.lockutils [req-dd2ca881-f580-4492-abe6-f6a81c22ff27 req-94d9af6b-344a-4dcf-819a-ecb0392b8255 service nova] Acquired lock "refresh_cache-d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 851.783929] env[64020]: DEBUG nova.network.neutron [req-dd2ca881-f580-4492-abe6-f6a81c22ff27 req-94d9af6b-344a-4dcf-819a-ecb0392b8255 service nova] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Refreshing network info cache for port 6a31c3b4-3e9f-44ef-a054-d0e46cbcd727 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 851.784350] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-dd2ca881-f580-4492-abe6-f6a81c22ff27 req-94d9af6b-344a-4dcf-819a-ecb0392b8255 service nova] Expecting reply to msg 5c115ac40da14492b0fba46bb4740cca in queue reply_57893177120949e6a93cb88e15cd42b4 [ 851.786338] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8cd412ba-4179-4846-a5e5-048bb589b8bf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.797165] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5c115ac40da14492b0fba46bb4740cca [ 851.799399] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6554ea65a73b41cb994faf7d243bb33a [ 851.802255] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45b4977d-dd90-4952-959a-0e7231aa6b58 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.827193] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d567d3dd-5820-4d6c-8922-3e39a1d9ebd6 could not be found. [ 851.827426] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 851.827601] env[64020]: INFO nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Took 0.04 seconds to destroy the instance on the hypervisor. [ 851.827845] env[64020]: DEBUG oslo.service.loopingcall [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 851.830244] env[64020]: DEBUG nova.compute.manager [-] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 851.830344] env[64020]: DEBUG nova.network.neutron [-] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 851.898789] env[64020]: DEBUG nova.compute.manager [None req-8570e1dc-d262-40fb-9e7b-ec34fd9a6d77 tempest-ServerDiagnosticsV248Test-610054911 tempest-ServerDiagnosticsV248Test-610054911-project-admin] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 851.900199] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-042e316d-be14-495c-9088-4e778092dd54 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.906575] env[64020]: INFO nova.compute.manager [None req-8570e1dc-d262-40fb-9e7b-ec34fd9a6d77 tempest-ServerDiagnosticsV248Test-610054911 tempest-ServerDiagnosticsV248Test-610054911-project-admin] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Retrieving diagnostics [ 851.909355] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a17fb75-2e3e-46d4-a590-c7a17ff157ea {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.912825] env[64020]: DEBUG nova.network.neutron [-] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 852.155631] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f53f4057-a977-4dfd-afba-06b4d77b4cd8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.165947] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab36e7c3-f6c9-4918-81ae-9582179629eb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.197733] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d97d85d1-02e4-4be6-97ec-8707d2d14160 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.200611] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6129d30-dedd-4c26-a6e8-d1392a0ce5ec tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Lock "995288b5-d79a-4af3-a1e8-3571fff2d356" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 158.958s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 852.201563] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 216b8b2265c443e5a1d1bc4480a317e0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 852.205693] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-581778ac-2e95-4869-8dd0-c92ea0125c9b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.218612] env[64020]: DEBUG nova.compute.provider_tree [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 852.219145] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg d7dabc4a68e8420081975fc5a4cd73d0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 852.220180] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 216b8b2265c443e5a1d1bc4480a317e0 [ 852.226390] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7dabc4a68e8420081975fc5a4cd73d0 [ 852.270371] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg a8d872373c174b2e84456d6c7f08e717 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 852.275042] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 418ccdc01f804668bd56e08d3f6d48bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 852.281498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 418ccdc01f804668bd56e08d3f6d48bc [ 852.304742] env[64020]: DEBUG nova.network.neutron [req-dd2ca881-f580-4492-abe6-f6a81c22ff27 req-94d9af6b-344a-4dcf-819a-ecb0392b8255 service nova] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 852.306736] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8d872373c174b2e84456d6c7f08e717 [ 852.385666] env[64020]: DEBUG nova.network.neutron [req-dd2ca881-f580-4492-abe6-f6a81c22ff27 req-94d9af6b-344a-4dcf-819a-ecb0392b8255 service nova] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 852.386207] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-dd2ca881-f580-4492-abe6-f6a81c22ff27 req-94d9af6b-344a-4dcf-819a-ecb0392b8255 service nova] Expecting reply to msg eefd810809b7487b976cbdb496ee4fa5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 852.393487] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eefd810809b7487b976cbdb496ee4fa5 [ 852.705530] env[64020]: DEBUG nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 852.707384] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 219d7998abcb4b4a858f0f377f87811e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 852.721641] env[64020]: DEBUG nova.scheduler.client.report [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 852.723964] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 0bc4543a6b764b9787aa9f504d7268f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 852.737918] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0bc4543a6b764b9787aa9f504d7268f5 [ 852.744238] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 219d7998abcb4b4a858f0f377f87811e [ 852.773299] env[64020]: DEBUG nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 852.776819] env[64020]: DEBUG nova.network.neutron [-] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 852.777146] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 43c556e4a5884b7e99db06a03df60347 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 852.793191] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43c556e4a5884b7e99db06a03df60347 [ 852.800582] env[64020]: DEBUG nova.virt.hardware [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 852.800714] env[64020]: DEBUG nova.virt.hardware [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 852.800883] env[64020]: DEBUG nova.virt.hardware [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 852.801073] env[64020]: DEBUG nova.virt.hardware [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 852.801253] env[64020]: DEBUG nova.virt.hardware [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 852.801405] env[64020]: DEBUG nova.virt.hardware [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 852.801625] env[64020]: DEBUG nova.virt.hardware [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 852.801794] env[64020]: DEBUG nova.virt.hardware [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 852.801962] env[64020]: DEBUG nova.virt.hardware [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 852.802122] env[64020]: DEBUG nova.virt.hardware [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 852.802303] env[64020]: DEBUG nova.virt.hardware [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 852.803443] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-883da2d3-4008-4c85-9500-82eed30df33f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.811907] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8dd78b9-d300-46bd-8d19-940ab199e695 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.889749] env[64020]: DEBUG oslo_concurrency.lockutils [req-dd2ca881-f580-4492-abe6-f6a81c22ff27 req-94d9af6b-344a-4dcf-819a-ecb0392b8255 service nova] Releasing lock "refresh_cache-d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 852.889976] env[64020]: DEBUG nova.compute.manager [req-dd2ca881-f580-4492-abe6-f6a81c22ff27 req-94d9af6b-344a-4dcf-819a-ecb0392b8255 service nova] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Received event network-vif-deleted-6a31c3b4-3e9f-44ef-a054-d0e46cbcd727 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 852.914549] env[64020]: DEBUG nova.compute.manager [req-b3b0d01c-4c92-44fd-bd0a-db3ac4ba972d req-5e7635e4-223d-48fa-957a-267983169de8 service nova] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Received event network-changed-53782645-b8fb-4e78-bdaf-8f397474785d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 852.914823] env[64020]: DEBUG nova.compute.manager [req-b3b0d01c-4c92-44fd-bd0a-db3ac4ba972d req-5e7635e4-223d-48fa-957a-267983169de8 service nova] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Refreshing instance network info cache due to event network-changed-53782645-b8fb-4e78-bdaf-8f397474785d. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 852.915105] env[64020]: DEBUG oslo_concurrency.lockutils [req-b3b0d01c-4c92-44fd-bd0a-db3ac4ba972d req-5e7635e4-223d-48fa-957a-267983169de8 service nova] Acquiring lock "refresh_cache-ecb7f281-1206-4693-845b-c59f639b0789" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 852.915297] env[64020]: DEBUG oslo_concurrency.lockutils [req-b3b0d01c-4c92-44fd-bd0a-db3ac4ba972d req-5e7635e4-223d-48fa-957a-267983169de8 service nova] Acquired lock "refresh_cache-ecb7f281-1206-4693-845b-c59f639b0789" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 852.915651] env[64020]: DEBUG nova.network.neutron [req-b3b0d01c-4c92-44fd-bd0a-db3ac4ba972d req-5e7635e4-223d-48fa-957a-267983169de8 service nova] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Refreshing network info cache for port 53782645-b8fb-4e78-bdaf-8f397474785d {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 852.916044] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-b3b0d01c-4c92-44fd-bd0a-db3ac4ba972d req-5e7635e4-223d-48fa-957a-267983169de8 service nova] Expecting reply to msg ac837edcefc84175a6a6cbfcc422bb21 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 852.945762] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ac837edcefc84175a6a6cbfcc422bb21 [ 853.044192] env[64020]: ERROR nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 53782645-b8fb-4e78-bdaf-8f397474785d, please check neutron logs for more information. [ 853.044192] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 853.044192] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 853.044192] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 853.044192] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 853.044192] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 853.044192] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 853.044192] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 853.044192] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 853.044192] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 853.044192] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 853.044192] env[64020]: ERROR nova.compute.manager raise self.value [ 853.044192] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 853.044192] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 853.044192] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 853.044192] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 853.044620] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 853.044620] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 853.044620] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 53782645-b8fb-4e78-bdaf-8f397474785d, please check neutron logs for more information. [ 853.044620] env[64020]: ERROR nova.compute.manager [ 853.044620] env[64020]: Traceback (most recent call last): [ 853.044620] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 853.044620] env[64020]: listener.cb(fileno) [ 853.044620] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 853.044620] env[64020]: result = function(*args, **kwargs) [ 853.044620] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 853.044620] env[64020]: return func(*args, **kwargs) [ 853.044620] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 853.044620] env[64020]: raise e [ 853.044620] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 853.044620] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 853.044620] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 853.044620] env[64020]: created_port_ids = self._update_ports_for_instance( [ 853.044620] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 853.044620] env[64020]: with excutils.save_and_reraise_exception(): [ 853.044620] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 853.044620] env[64020]: self.force_reraise() [ 853.044620] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 853.044620] env[64020]: raise self.value [ 853.044620] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 853.044620] env[64020]: updated_port = self._update_port( [ 853.044620] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 853.044620] env[64020]: _ensure_no_port_binding_failure(port) [ 853.044620] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 853.044620] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 853.045244] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 53782645-b8fb-4e78-bdaf-8f397474785d, please check neutron logs for more information. [ 853.045244] env[64020]: Removing descriptor: 16 [ 853.045244] env[64020]: ERROR nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 53782645-b8fb-4e78-bdaf-8f397474785d, please check neutron logs for more information. [ 853.045244] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] Traceback (most recent call last): [ 853.045244] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 853.045244] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] yield resources [ 853.045244] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 853.045244] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] self.driver.spawn(context, instance, image_meta, [ 853.045244] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 853.045244] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] self._vmops.spawn(context, instance, image_meta, injected_files, [ 853.045244] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 853.045244] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] vm_ref = self.build_virtual_machine(instance, [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] vif_infos = vmwarevif.get_vif_info(self._session, [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] for vif in network_info: [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] return self._sync_wrapper(fn, *args, **kwargs) [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] self.wait() [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] self[:] = self._gt.wait() [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] return self._exit_event.wait() [ 853.045507] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] result = hub.switch() [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] return self.greenlet.switch() [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] result = function(*args, **kwargs) [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] return func(*args, **kwargs) [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] raise e [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] nwinfo = self.network_api.allocate_for_instance( [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 853.045781] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] created_port_ids = self._update_ports_for_instance( [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] with excutils.save_and_reraise_exception(): [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] self.force_reraise() [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] raise self.value [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] updated_port = self._update_port( [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] _ensure_no_port_binding_failure(port) [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 853.046055] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] raise exception.PortBindingFailed(port_id=port['id']) [ 853.046314] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] nova.exception.PortBindingFailed: Binding failed for port 53782645-b8fb-4e78-bdaf-8f397474785d, please check neutron logs for more information. [ 853.046314] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] [ 853.046314] env[64020]: INFO nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Terminating instance [ 853.047274] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-ecb7f281-1206-4693-845b-c59f639b0789" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 853.226371] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.472s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 853.226859] env[64020]: DEBUG nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 853.228502] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg e8225df330944b039778fbdbbc8d3685 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 853.230742] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 853.230994] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 28.682s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 853.232440] env[64020]: INFO nova.compute.claims [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 853.234031] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 8fafc5b8e1d542a1bb058f6eabceb98f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 853.257549] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e8225df330944b039778fbdbbc8d3685 [ 853.267039] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8fafc5b8e1d542a1bb058f6eabceb98f [ 853.278394] env[64020]: INFO nova.compute.manager [-] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Took 1.45 seconds to deallocate network for instance. [ 853.280383] env[64020]: DEBUG nova.compute.claims [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 853.280555] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 853.448038] env[64020]: DEBUG nova.network.neutron [req-b3b0d01c-4c92-44fd-bd0a-db3ac4ba972d req-5e7635e4-223d-48fa-957a-267983169de8 service nova] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 853.591856] env[64020]: DEBUG nova.network.neutron [req-b3b0d01c-4c92-44fd-bd0a-db3ac4ba972d req-5e7635e4-223d-48fa-957a-267983169de8 service nova] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 853.592501] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-b3b0d01c-4c92-44fd-bd0a-db3ac4ba972d req-5e7635e4-223d-48fa-957a-267983169de8 service nova] Expecting reply to msg 53e85b52f9bc4a02beda55843b40f168 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 853.600754] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 53e85b52f9bc4a02beda55843b40f168 [ 853.731743] env[64020]: DEBUG nova.compute.utils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 853.732403] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg e2d36b0e41ca4c72acd08615f1b25b5f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 853.733580] env[64020]: DEBUG nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 853.733761] env[64020]: DEBUG nova.network.neutron [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 853.737016] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg baef3f29079842b8b426fb80c8607373 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 853.743460] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e2d36b0e41ca4c72acd08615f1b25b5f [ 853.745394] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg baef3f29079842b8b426fb80c8607373 [ 853.802701] env[64020]: DEBUG nova.policy [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '1c1610f2158f4501bda3ba14d5282882', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8da409b9607348ddb9edd6e6579e5f32', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 854.092794] env[64020]: DEBUG nova.network.neutron [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Successfully created port: 7781f67c-a182-4566-a351-61cb71e27082 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 854.095358] env[64020]: DEBUG oslo_concurrency.lockutils [req-b3b0d01c-4c92-44fd-bd0a-db3ac4ba972d req-5e7635e4-223d-48fa-957a-267983169de8 service nova] Releasing lock "refresh_cache-ecb7f281-1206-4693-845b-c59f639b0789" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 854.095793] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-ecb7f281-1206-4693-845b-c59f639b0789" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 854.096017] env[64020]: DEBUG nova.network.neutron [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 854.096466] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 3580e45644524c08a7d3e073a56daa4a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 854.105188] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3580e45644524c08a7d3e073a56daa4a [ 854.236500] env[64020]: DEBUG nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 854.238176] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 460aec8b5b814a39850abec90630571b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 854.267215] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 460aec8b5b814a39850abec90630571b [ 854.607119] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d4134f6-54a7-4094-8bf1-4326e5df27f8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 854.614744] env[64020]: DEBUG nova.network.neutron [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 854.617065] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fad50cab-cefb-4af4-9122-13f312c4314d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 854.652710] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70af88da-f964-4328-9cd3-97b1b0a63208 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 854.664527] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-623690bd-ccbf-4cd3-b387-046aefa7e628 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 854.678987] env[64020]: DEBUG nova.compute.provider_tree [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 854.679865] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 12fd2b7fd7e149849d8c2efe71ade42d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 854.687356] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 12fd2b7fd7e149849d8c2efe71ade42d [ 854.703163] env[64020]: DEBUG nova.network.neutron [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 854.703698] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 2691f7af560a4755ada5534c495654b0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 854.713009] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2691f7af560a4755ada5534c495654b0 [ 854.745305] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg ab27f0fd7da943738dc80557417db6e8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 854.778780] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab27f0fd7da943738dc80557417db6e8 [ 854.971697] env[64020]: DEBUG nova.compute.manager [req-4e745caa-0221-409a-afe6-4e25b8d295a4 req-be0bfdc6-eb41-4a69-b64f-109a0ed80d0f service nova] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Received event network-vif-deleted-53782645-b8fb-4e78-bdaf-8f397474785d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 855.182358] env[64020]: DEBUG nova.scheduler.client.report [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 855.188237] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 8c495df0752749dcb04078f295e0da9f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 855.199582] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8c495df0752749dcb04078f295e0da9f [ 855.203854] env[64020]: ERROR nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 7781f67c-a182-4566-a351-61cb71e27082, please check neutron logs for more information. [ 855.203854] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 855.203854] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 855.203854] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 855.203854] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 855.203854] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 855.203854] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 855.203854] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 855.203854] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 855.203854] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 855.203854] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 855.203854] env[64020]: ERROR nova.compute.manager raise self.value [ 855.203854] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 855.203854] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 855.203854] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 855.203854] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 855.204298] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 855.204298] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 855.204298] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 7781f67c-a182-4566-a351-61cb71e27082, please check neutron logs for more information. [ 855.204298] env[64020]: ERROR nova.compute.manager [ 855.204440] env[64020]: Traceback (most recent call last): [ 855.204501] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 855.204501] env[64020]: listener.cb(fileno) [ 855.204501] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 855.204501] env[64020]: result = function(*args, **kwargs) [ 855.204501] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 855.204501] env[64020]: return func(*args, **kwargs) [ 855.204650] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 855.204650] env[64020]: raise e [ 855.204650] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 855.204650] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 855.204650] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 855.204650] env[64020]: created_port_ids = self._update_ports_for_instance( [ 855.204650] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 855.204650] env[64020]: with excutils.save_and_reraise_exception(): [ 855.204650] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 855.204650] env[64020]: self.force_reraise() [ 855.204650] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 855.204650] env[64020]: raise self.value [ 855.204650] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 855.204650] env[64020]: updated_port = self._update_port( [ 855.204650] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 855.204650] env[64020]: _ensure_no_port_binding_failure(port) [ 855.204650] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 855.204650] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 855.204650] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 7781f67c-a182-4566-a351-61cb71e27082, please check neutron logs for more information. [ 855.204650] env[64020]: Removing descriptor: 16 [ 855.205682] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-ecb7f281-1206-4693-845b-c59f639b0789" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 855.206164] env[64020]: DEBUG nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 855.206446] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 855.206794] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8e00f737-1bda-4ef1-bb06-40c7ea3e5a8c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.222905] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28e3a448-2876-4c65-9ecd-6f44817bd93d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.245409] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ecb7f281-1206-4693-845b-c59f639b0789 could not be found. [ 855.245666] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 855.245960] env[64020]: INFO nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Took 0.04 seconds to destroy the instance on the hypervisor. [ 855.246308] env[64020]: DEBUG oslo.service.loopingcall [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 855.246582] env[64020]: DEBUG nova.compute.manager [-] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 855.246735] env[64020]: DEBUG nova.network.neutron [-] [instance: ecb7f281-1206-4693-845b-c59f639b0789] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 855.249445] env[64020]: DEBUG nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 855.264798] env[64020]: DEBUG nova.network.neutron [-] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 855.265328] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 04d4f8a8b0f7410885310b4f24708900 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 855.275804] env[64020]: DEBUG nova.virt.hardware [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 855.275804] env[64020]: DEBUG nova.virt.hardware [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 855.275804] env[64020]: DEBUG nova.virt.hardware [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 855.276444] env[64020]: DEBUG nova.virt.hardware [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 855.276444] env[64020]: DEBUG nova.virt.hardware [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 855.276444] env[64020]: DEBUG nova.virt.hardware [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 855.276672] env[64020]: DEBUG nova.virt.hardware [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 855.276891] env[64020]: DEBUG nova.virt.hardware [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 855.277126] env[64020]: DEBUG nova.virt.hardware [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 855.277348] env[64020]: DEBUG nova.virt.hardware [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 855.277583] env[64020]: DEBUG nova.virt.hardware [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 855.278432] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b220ca3-c521-4583-8416-1e45649cb863 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.281273] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 04d4f8a8b0f7410885310b4f24708900 [ 855.286999] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d00534b1-82d4-4cf6-9f80-215d74ecfba4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.300759] env[64020]: ERROR nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 7781f67c-a182-4566-a351-61cb71e27082, please check neutron logs for more information. [ 855.300759] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Traceback (most recent call last): [ 855.300759] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 855.300759] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] yield resources [ 855.300759] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 855.300759] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] self.driver.spawn(context, instance, image_meta, [ 855.300759] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 855.300759] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 855.300759] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 855.300759] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] vm_ref = self.build_virtual_machine(instance, [ 855.300759] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] vif_infos = vmwarevif.get_vif_info(self._session, [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] for vif in network_info: [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] return self._sync_wrapper(fn, *args, **kwargs) [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] self.wait() [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] self[:] = self._gt.wait() [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] return self._exit_event.wait() [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 855.301160] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] current.throw(*self._exc) [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] result = function(*args, **kwargs) [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] return func(*args, **kwargs) [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] raise e [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] nwinfo = self.network_api.allocate_for_instance( [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] created_port_ids = self._update_ports_for_instance( [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] with excutils.save_and_reraise_exception(): [ 855.301524] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 855.301846] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] self.force_reraise() [ 855.301846] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 855.301846] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] raise self.value [ 855.301846] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 855.301846] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] updated_port = self._update_port( [ 855.301846] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 855.301846] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] _ensure_no_port_binding_failure(port) [ 855.301846] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 855.301846] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] raise exception.PortBindingFailed(port_id=port['id']) [ 855.301846] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] nova.exception.PortBindingFailed: Binding failed for port 7781f67c-a182-4566-a351-61cb71e27082, please check neutron logs for more information. [ 855.301846] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] [ 855.301846] env[64020]: INFO nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Terminating instance [ 855.303999] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquiring lock "refresh_cache-8de44ad9-adb7-4589-b2d8-a2aeeb89892f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 855.304234] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquired lock "refresh_cache-8de44ad9-adb7-4589-b2d8-a2aeeb89892f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 855.304458] env[64020]: DEBUG nova.network.neutron [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 855.304909] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg e3334f88a8a54d9aa510bc4cee993046 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 855.310768] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3334f88a8a54d9aa510bc4cee993046 [ 855.690742] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.460s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 855.691337] env[64020]: DEBUG nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 855.693327] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg b8291bd58b904081bf9054a7700c0444 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 855.694397] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 29.472s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 855.704736] env[64020]: DEBUG nova.objects.instance [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] [instance: e27f988a-a64b-407b-ad66-4bed7ee45b52] Trying to apply a migration context that does not seem to be set for this instance {{(pid=64020) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 855.704736] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 4f0d60819a11449dbf4839a6d5fca485 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 855.732852] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f0d60819a11449dbf4839a6d5fca485 [ 855.736805] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8291bd58b904081bf9054a7700c0444 [ 855.767318] env[64020]: DEBUG nova.network.neutron [-] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 855.767735] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7ea8011487084fb3af52e135a0b616b8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 855.775047] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ea8011487084fb3af52e135a0b616b8 [ 856.502834] env[64020]: DEBUG nova.compute.utils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 856.503435] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 3c3aee08388f41c4b2e07f43fe05fc5a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 856.506162] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 0ada668ed0f84e5f8c1eb75c58044963 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 856.511752] env[64020]: INFO nova.compute.manager [-] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Took 1.26 seconds to deallocate network for instance. [ 856.511752] env[64020]: DEBUG nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 856.511752] env[64020]: DEBUG nova.network.neutron [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 856.511752] env[64020]: DEBUG nova.compute.claims [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 856.511752] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 856.517408] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0ada668ed0f84e5f8c1eb75c58044963 [ 856.517998] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c3aee08388f41c4b2e07f43fe05fc5a [ 856.523444] env[64020]: DEBUG nova.network.neutron [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 856.550598] env[64020]: DEBUG nova.policy [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e6417779415348f2b74727cf41c77c54', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'fbebd163ed084de0a8a9f6ccc620ab6a', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 856.625067] env[64020]: DEBUG nova.network.neutron [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 856.625647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg f2c2fcd4f2324c6caaacd54d853d9934 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 856.633727] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f2c2fcd4f2324c6caaacd54d853d9934 [ 856.857564] env[64020]: DEBUG nova.network.neutron [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Successfully created port: d6820d39-d605-4caf-a814-ff1498a867f6 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 856.998464] env[64020]: DEBUG nova.compute.manager [req-6365f5ae-c8e6-4e99-8ba0-bc0f8850d539 req-5aeb7fb9-35cd-462e-a420-253bee8fa4ba service nova] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Received event network-changed-7781f67c-a182-4566-a351-61cb71e27082 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 856.998676] env[64020]: DEBUG nova.compute.manager [req-6365f5ae-c8e6-4e99-8ba0-bc0f8850d539 req-5aeb7fb9-35cd-462e-a420-253bee8fa4ba service nova] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Refreshing instance network info cache due to event network-changed-7781f67c-a182-4566-a351-61cb71e27082. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 856.998818] env[64020]: DEBUG oslo_concurrency.lockutils [req-6365f5ae-c8e6-4e99-8ba0-bc0f8850d539 req-5aeb7fb9-35cd-462e-a420-253bee8fa4ba service nova] Acquiring lock "refresh_cache-8de44ad9-adb7-4589-b2d8-a2aeeb89892f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 857.008366] env[64020]: DEBUG nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 857.010133] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 3fe86633139b4ab495da61be796e2f11 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 857.011882] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.317s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 857.012331] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a08d024c-a0ca-4781-bffd-f3553a309be9 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 47825b71f0f74bd8bff365ed96f2dadd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 857.013248] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 27.934s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 857.014928] env[64020]: INFO nova.compute.claims [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 857.016411] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg b8da377c99484944ae7aea0bd0d1c988 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 857.027447] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47825b71f0f74bd8bff365ed96f2dadd [ 857.056318] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8da377c99484944ae7aea0bd0d1c988 [ 857.078719] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3fe86633139b4ab495da61be796e2f11 [ 857.128205] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Releasing lock "refresh_cache-8de44ad9-adb7-4589-b2d8-a2aeeb89892f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 857.128657] env[64020]: DEBUG nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 857.128812] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 857.129119] env[64020]: DEBUG oslo_concurrency.lockutils [req-6365f5ae-c8e6-4e99-8ba0-bc0f8850d539 req-5aeb7fb9-35cd-462e-a420-253bee8fa4ba service nova] Acquired lock "refresh_cache-8de44ad9-adb7-4589-b2d8-a2aeeb89892f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 857.129287] env[64020]: DEBUG nova.network.neutron [req-6365f5ae-c8e6-4e99-8ba0-bc0f8850d539 req-5aeb7fb9-35cd-462e-a420-253bee8fa4ba service nova] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Refreshing network info cache for port 7781f67c-a182-4566-a351-61cb71e27082 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 857.129722] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-6365f5ae-c8e6-4e99-8ba0-bc0f8850d539 req-5aeb7fb9-35cd-462e-a420-253bee8fa4ba service nova] Expecting reply to msg 143b5d8923b545aeacd64c26ad9bbd9e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 857.130493] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-06de84c4-fbe7-4351-9072-626a7c9e74b2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.136206] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 143b5d8923b545aeacd64c26ad9bbd9e [ 857.140254] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2bc8f74-e578-426e-9781-5c61383d1efb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.161374] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 8de44ad9-adb7-4589-b2d8-a2aeeb89892f could not be found. [ 857.161580] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 857.161757] env[64020]: INFO nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Took 0.03 seconds to destroy the instance on the hypervisor. [ 857.161988] env[64020]: DEBUG oslo.service.loopingcall [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 857.162213] env[64020]: DEBUG nova.compute.manager [-] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 857.162298] env[64020]: DEBUG nova.network.neutron [-] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 857.181427] env[64020]: DEBUG nova.network.neutron [-] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 857.181897] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 38019218bd944a7a9440446ae079c3c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 857.188683] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 38019218bd944a7a9440446ae079c3c8 [ 857.499547] env[64020]: DEBUG nova.compute.manager [req-44b97dca-b584-4fae-a1b6-cf47cd2407a8 req-c711bdc5-3c9c-48b4-a268-9b5a826584ff service nova] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Received event network-changed-d6820d39-d605-4caf-a814-ff1498a867f6 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 857.499764] env[64020]: DEBUG nova.compute.manager [req-44b97dca-b584-4fae-a1b6-cf47cd2407a8 req-c711bdc5-3c9c-48b4-a268-9b5a826584ff service nova] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Refreshing instance network info cache due to event network-changed-d6820d39-d605-4caf-a814-ff1498a867f6. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 857.499971] env[64020]: DEBUG oslo_concurrency.lockutils [req-44b97dca-b584-4fae-a1b6-cf47cd2407a8 req-c711bdc5-3c9c-48b4-a268-9b5a826584ff service nova] Acquiring lock "refresh_cache-f7f8d239-aff8-4cd7-afdd-974c53b3e563" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 857.500088] env[64020]: DEBUG oslo_concurrency.lockutils [req-44b97dca-b584-4fae-a1b6-cf47cd2407a8 req-c711bdc5-3c9c-48b4-a268-9b5a826584ff service nova] Acquired lock "refresh_cache-f7f8d239-aff8-4cd7-afdd-974c53b3e563" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 857.500252] env[64020]: DEBUG nova.network.neutron [req-44b97dca-b584-4fae-a1b6-cf47cd2407a8 req-c711bdc5-3c9c-48b4-a268-9b5a826584ff service nova] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Refreshing network info cache for port d6820d39-d605-4caf-a814-ff1498a867f6 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 857.500658] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-44b97dca-b584-4fae-a1b6-cf47cd2407a8 req-c711bdc5-3c9c-48b4-a268-9b5a826584ff service nova] Expecting reply to msg d92c32f367d44635a60f6988b36d40ee in queue reply_57893177120949e6a93cb88e15cd42b4 [ 857.506733] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d92c32f367d44635a60f6988b36d40ee [ 857.516129] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 0a24b54bb06b4175b0c00ebafb03b5c9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 857.519366] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg b9cdd895ccb44cd99d41aa3a24971fbd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 857.525811] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9cdd895ccb44cd99d41aa3a24971fbd [ 857.546820] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a24b54bb06b4175b0c00ebafb03b5c9 [ 857.658628] env[64020]: DEBUG nova.network.neutron [req-6365f5ae-c8e6-4e99-8ba0-bc0f8850d539 req-5aeb7fb9-35cd-462e-a420-253bee8fa4ba service nova] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 857.683362] env[64020]: ERROR nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d6820d39-d605-4caf-a814-ff1498a867f6, please check neutron logs for more information. [ 857.683362] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 857.683362] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 857.683362] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 857.683362] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 857.683362] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 857.683362] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 857.683362] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 857.683362] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 857.683362] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 857.683362] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 857.683362] env[64020]: ERROR nova.compute.manager raise self.value [ 857.683362] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 857.683362] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 857.683362] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 857.683362] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 857.683766] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 857.683766] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 857.683766] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d6820d39-d605-4caf-a814-ff1498a867f6, please check neutron logs for more information. [ 857.683766] env[64020]: ERROR nova.compute.manager [ 857.683766] env[64020]: Traceback (most recent call last): [ 857.683766] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 857.683766] env[64020]: listener.cb(fileno) [ 857.683766] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 857.683766] env[64020]: result = function(*args, **kwargs) [ 857.683766] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 857.683766] env[64020]: return func(*args, **kwargs) [ 857.683766] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 857.683766] env[64020]: raise e [ 857.683766] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 857.683766] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 857.683766] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 857.683766] env[64020]: created_port_ids = self._update_ports_for_instance( [ 857.683766] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 857.683766] env[64020]: with excutils.save_and_reraise_exception(): [ 857.683766] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 857.683766] env[64020]: self.force_reraise() [ 857.683766] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 857.683766] env[64020]: raise self.value [ 857.683766] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 857.683766] env[64020]: updated_port = self._update_port( [ 857.683766] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 857.683766] env[64020]: _ensure_no_port_binding_failure(port) [ 857.683766] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 857.683766] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 857.684407] env[64020]: nova.exception.PortBindingFailed: Binding failed for port d6820d39-d605-4caf-a814-ff1498a867f6, please check neutron logs for more information. [ 857.684407] env[64020]: Removing descriptor: 18 [ 857.684407] env[64020]: DEBUG nova.network.neutron [-] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 857.684407] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 92db03d9942541fab9ddfddb997defb5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 857.692169] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92db03d9942541fab9ddfddb997defb5 [ 857.761968] env[64020]: DEBUG nova.network.neutron [req-6365f5ae-c8e6-4e99-8ba0-bc0f8850d539 req-5aeb7fb9-35cd-462e-a420-253bee8fa4ba service nova] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 857.762520] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-6365f5ae-c8e6-4e99-8ba0-bc0f8850d539 req-5aeb7fb9-35cd-462e-a420-253bee8fa4ba service nova] Expecting reply to msg ae003edc5fea44378e03bd6d250f3b3f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 857.770278] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae003edc5fea44378e03bd6d250f3b3f [ 858.019749] env[64020]: DEBUG nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 858.023406] env[64020]: DEBUG nova.network.neutron [req-44b97dca-b584-4fae-a1b6-cf47cd2407a8 req-c711bdc5-3c9c-48b4-a268-9b5a826584ff service nova] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 858.052897] env[64020]: DEBUG nova.virt.hardware [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 858.053046] env[64020]: DEBUG nova.virt.hardware [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 858.053162] env[64020]: DEBUG nova.virt.hardware [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 858.053371] env[64020]: DEBUG nova.virt.hardware [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 858.054018] env[64020]: DEBUG nova.virt.hardware [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 858.054018] env[64020]: DEBUG nova.virt.hardware [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 858.054018] env[64020]: DEBUG nova.virt.hardware [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 858.054171] env[64020]: DEBUG nova.virt.hardware [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 858.054298] env[64020]: DEBUG nova.virt.hardware [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 858.054455] env[64020]: DEBUG nova.virt.hardware [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 858.054616] env[64020]: DEBUG nova.virt.hardware [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 858.056115] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fcb01486-f01d-4a66-b7d4-cd8a59f81b42 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 858.067051] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97efac70-1c8f-408d-a9e0-bb150896deb5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 858.084419] env[64020]: ERROR nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d6820d39-d605-4caf-a814-ff1498a867f6, please check neutron logs for more information. [ 858.084419] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Traceback (most recent call last): [ 858.084419] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 858.084419] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] yield resources [ 858.084419] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 858.084419] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] self.driver.spawn(context, instance, image_meta, [ 858.084419] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 858.084419] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] self._vmops.spawn(context, instance, image_meta, injected_files, [ 858.084419] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 858.084419] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] vm_ref = self.build_virtual_machine(instance, [ 858.084419] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] vif_infos = vmwarevif.get_vif_info(self._session, [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] for vif in network_info: [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] return self._sync_wrapper(fn, *args, **kwargs) [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] self.wait() [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] self[:] = self._gt.wait() [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] return self._exit_event.wait() [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 858.084754] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] current.throw(*self._exc) [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] result = function(*args, **kwargs) [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] return func(*args, **kwargs) [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] raise e [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] nwinfo = self.network_api.allocate_for_instance( [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] created_port_ids = self._update_ports_for_instance( [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] with excutils.save_and_reraise_exception(): [ 858.085066] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 858.085446] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] self.force_reraise() [ 858.085446] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 858.085446] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] raise self.value [ 858.085446] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 858.085446] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] updated_port = self._update_port( [ 858.085446] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 858.085446] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] _ensure_no_port_binding_failure(port) [ 858.085446] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 858.085446] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] raise exception.PortBindingFailed(port_id=port['id']) [ 858.085446] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] nova.exception.PortBindingFailed: Binding failed for port d6820d39-d605-4caf-a814-ff1498a867f6, please check neutron logs for more information. [ 858.085446] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] [ 858.085446] env[64020]: INFO nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Terminating instance [ 858.088703] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquiring lock "refresh_cache-f7f8d239-aff8-4cd7-afdd-974c53b3e563" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 858.131798] env[64020]: DEBUG nova.network.neutron [req-44b97dca-b584-4fae-a1b6-cf47cd2407a8 req-c711bdc5-3c9c-48b4-a268-9b5a826584ff service nova] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 858.132362] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-44b97dca-b584-4fae-a1b6-cf47cd2407a8 req-c711bdc5-3c9c-48b4-a268-9b5a826584ff service nova] Expecting reply to msg 55d82a32f8c8433d93cc2b69791406d2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 858.142323] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 55d82a32f8c8433d93cc2b69791406d2 [ 858.186360] env[64020]: INFO nova.compute.manager [-] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Took 1.02 seconds to deallocate network for instance. [ 858.190629] env[64020]: DEBUG nova.compute.claims [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 858.190795] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 858.265185] env[64020]: DEBUG oslo_concurrency.lockutils [req-6365f5ae-c8e6-4e99-8ba0-bc0f8850d539 req-5aeb7fb9-35cd-462e-a420-253bee8fa4ba service nova] Releasing lock "refresh_cache-8de44ad9-adb7-4589-b2d8-a2aeeb89892f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 858.265337] env[64020]: DEBUG nova.compute.manager [req-6365f5ae-c8e6-4e99-8ba0-bc0f8850d539 req-5aeb7fb9-35cd-462e-a420-253bee8fa4ba service nova] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Received event network-vif-deleted-7781f67c-a182-4566-a351-61cb71e27082 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 858.354523] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f59c7dfd-77e0-4a20-a126-f52ea32a763f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 858.362073] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d179c83a-fe95-44e6-9b6d-4451ffe47b5c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 858.391358] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5850fcaa-33a7-4ca8-92ac-58e1d830e9df {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 858.399418] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1909308c-869d-49fa-8d3b-ca3f690bfc18 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 858.414894] env[64020]: DEBUG nova.compute.provider_tree [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 858.415383] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg b2d67f348afe4fc6a4d551fc459b234b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 858.422601] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b2d67f348afe4fc6a4d551fc459b234b [ 858.634361] env[64020]: DEBUG oslo_concurrency.lockutils [req-44b97dca-b584-4fae-a1b6-cf47cd2407a8 req-c711bdc5-3c9c-48b4-a268-9b5a826584ff service nova] Releasing lock "refresh_cache-f7f8d239-aff8-4cd7-afdd-974c53b3e563" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 858.634817] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquired lock "refresh_cache-f7f8d239-aff8-4cd7-afdd-974c53b3e563" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 858.635081] env[64020]: DEBUG nova.network.neutron [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 858.635571] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg f5a575ec72714fa38b66f786c9f6dddb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 858.642854] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5a575ec72714fa38b66f786c9f6dddb [ 858.918882] env[64020]: DEBUG nova.scheduler.client.report [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 858.921448] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 0cc891540fe64a19af77cb80f52db945 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 858.934969] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0cc891540fe64a19af77cb80f52db945 [ 859.150306] env[64020]: DEBUG nova.network.neutron [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 859.232310] env[64020]: DEBUG nova.network.neutron [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 859.232809] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg fdca291c41714b538a1227cc959e7491 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 859.242083] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fdca291c41714b538a1227cc959e7491 [ 859.424253] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.411s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 859.424785] env[64020]: DEBUG nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 859.426818] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 1524e1a972e84c5f8730350d690866a5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 859.428979] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 29.139s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 859.430814] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 06b3f38335614ed3812310704c8fb4c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 859.459524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1524e1a972e84c5f8730350d690866a5 [ 859.461867] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06b3f38335614ed3812310704c8fb4c1 [ 859.524885] env[64020]: DEBUG nova.compute.manager [req-e44ba314-3e3a-4b9b-9901-db91118ad05b req-dd48d637-16a5-42af-8b3c-d1e6dea8e3ef service nova] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Received event network-vif-deleted-d6820d39-d605-4caf-a814-ff1498a867f6 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 859.735226] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Releasing lock "refresh_cache-f7f8d239-aff8-4cd7-afdd-974c53b3e563" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 859.735660] env[64020]: DEBUG nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 859.735844] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 859.736145] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2226b8df-9023-4b1b-a1bc-15b769315fc3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.745480] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e8b1261-95c2-4c6b-9e2c-65ad695f1442 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.767502] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f7f8d239-aff8-4cd7-afdd-974c53b3e563 could not be found. [ 859.767727] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 859.767904] env[64020]: INFO nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Took 0.03 seconds to destroy the instance on the hypervisor. [ 859.768150] env[64020]: DEBUG oslo.service.loopingcall [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 859.768364] env[64020]: DEBUG nova.compute.manager [-] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 859.768455] env[64020]: DEBUG nova.network.neutron [-] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 859.789945] env[64020]: DEBUG nova.network.neutron [-] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 859.790413] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 856c99fa08614f3bb7d5699a18a9ea2a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 859.798749] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 856c99fa08614f3bb7d5699a18a9ea2a [ 859.933950] env[64020]: DEBUG nova.compute.utils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 859.934612] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 8f1763c83b284f799280a957d1301fe5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 859.939432] env[64020]: DEBUG nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 859.940027] env[64020]: DEBUG nova.network.neutron [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 859.947853] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f1763c83b284f799280a957d1301fe5 [ 859.990618] env[64020]: DEBUG nova.policy [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '29faa85bd7904cd985fb32f53b533c5b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '525215939f6741cea749e7d9e53f379e', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 860.244789] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a16767f1-a674-4f29-a703-95d6ab4bfb9a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 860.252977] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6d68eb4-58a0-45a8-8c82-63dc5ad3f739 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 860.287196] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-901e8787-2060-4f57-9b15-cf673fed93ab {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 860.294211] env[64020]: DEBUG nova.network.neutron [-] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 860.294867] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 035ecd98853b41e386e5e117a8fba968 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 860.296813] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0a53e38-1f5f-449b-ba9a-9dc3dfb26c12 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 860.303765] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 035ecd98853b41e386e5e117a8fba968 [ 860.314213] env[64020]: DEBUG nova.compute.provider_tree [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 860.315008] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg fbdebd9bb7e347f4bc080dc49e601fb3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 860.324583] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fbdebd9bb7e347f4bc080dc49e601fb3 [ 860.325704] env[64020]: DEBUG nova.scheduler.client.report [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 860.330782] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg deb63649c2a7485ea1244cb5a2a9f105 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 860.339266] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg deb63649c2a7485ea1244cb5a2a9f105 [ 860.367864] env[64020]: DEBUG nova.network.neutron [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Successfully created port: 12c430b0-3ca9-4fe0-ad3d-30068930d0e7 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 860.443009] env[64020]: DEBUG nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 860.444751] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg bc709d6166bc442ba0e0a1b09b9da4c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 860.477035] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc709d6166bc442ba0e0a1b09b9da4c8 [ 860.805248] env[64020]: INFO nova.compute.manager [-] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Took 1.04 seconds to deallocate network for instance. [ 860.806334] env[64020]: DEBUG nova.compute.claims [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 860.806334] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 860.835987] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.402s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 860.835987] env[64020]: ERROR nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 112e5229-7ce7-4a48-8233-fa58ce776b1a, please check neutron logs for more information. [ 860.835987] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Traceback (most recent call last): [ 860.835987] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 860.835987] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] self.driver.spawn(context, instance, image_meta, [ 860.835987] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 860.835987] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 860.835987] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 860.835987] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] vm_ref = self.build_virtual_machine(instance, [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] vif_infos = vmwarevif.get_vif_info(self._session, [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] for vif in network_info: [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] return self._sync_wrapper(fn, *args, **kwargs) [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] self.wait() [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] self[:] = self._gt.wait() [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] return self._exit_event.wait() [ 860.836479] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] result = hub.switch() [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] return self.greenlet.switch() [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] result = function(*args, **kwargs) [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] return func(*args, **kwargs) [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] raise e [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] nwinfo = self.network_api.allocate_for_instance( [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 860.837113] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] created_port_ids = self._update_ports_for_instance( [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] with excutils.save_and_reraise_exception(): [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] self.force_reraise() [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] raise self.value [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] updated_port = self._update_port( [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] _ensure_no_port_binding_failure(port) [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 860.837628] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] raise exception.PortBindingFailed(port_id=port['id']) [ 860.838104] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] nova.exception.PortBindingFailed: Binding failed for port 112e5229-7ce7-4a48-8233-fa58ce776b1a, please check neutron logs for more information. [ 860.838104] env[64020]: ERROR nova.compute.manager [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] [ 860.838104] env[64020]: DEBUG nova.compute.utils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Binding failed for port 112e5229-7ce7-4a48-8233-fa58ce776b1a, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 860.838104] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 30.125s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 860.838104] env[64020]: DEBUG nova.objects.instance [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lazy-loading 'resources' on Instance uuid e27f988a-a64b-407b-ad66-4bed7ee45b52 {{(pid=64020) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 860.838104] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 8a0da28c309a4bf9ac32d30ef0206e87 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 860.843725] env[64020]: DEBUG nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Build of instance 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3 was re-scheduled: Binding failed for port 112e5229-7ce7-4a48-8233-fa58ce776b1a, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 860.843725] env[64020]: DEBUG nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 860.843725] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 860.843725] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 860.844096] env[64020]: DEBUG nova.network.neutron [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 860.844096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 92d18f15dab84e9598e1dba483abcee9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 860.844096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a0da28c309a4bf9ac32d30ef0206e87 [ 860.847756] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92d18f15dab84e9598e1dba483abcee9 [ 860.950539] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 2781e60a098342c8bb7a24dac5f8954a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 860.981261] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2781e60a098342c8bb7a24dac5f8954a [ 861.361898] env[64020]: DEBUG nova.network.neutron [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 861.442422] env[64020]: DEBUG nova.network.neutron [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 861.443163] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 038d49c640ee406387c1c8e471117af1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 861.458049] env[64020]: DEBUG nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 861.462701] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 038d49c640ee406387c1c8e471117af1 [ 861.487828] env[64020]: DEBUG nova.virt.hardware [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 861.488084] env[64020]: DEBUG nova.virt.hardware [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 861.488246] env[64020]: DEBUG nova.virt.hardware [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 861.488426] env[64020]: DEBUG nova.virt.hardware [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 861.488570] env[64020]: DEBUG nova.virt.hardware [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 861.488738] env[64020]: DEBUG nova.virt.hardware [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 861.488907] env[64020]: DEBUG nova.virt.hardware [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 861.489071] env[64020]: DEBUG nova.virt.hardware [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 861.489261] env[64020]: DEBUG nova.virt.hardware [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 861.489424] env[64020]: DEBUG nova.virt.hardware [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 861.489593] env[64020]: DEBUG nova.virt.hardware [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 861.490669] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94edb1bc-4ad3-4868-a3a2-4f00505ee2eb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.494799] env[64020]: DEBUG nova.compute.manager [req-a7e4de68-d64c-48ba-bb08-6d86f8950593 req-eb04ac33-ecb8-47a3-8069-df48e3277328 service nova] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Received event network-changed-12c430b0-3ca9-4fe0-ad3d-30068930d0e7 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 861.500112] env[64020]: DEBUG nova.compute.manager [req-a7e4de68-d64c-48ba-bb08-6d86f8950593 req-eb04ac33-ecb8-47a3-8069-df48e3277328 service nova] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Refreshing instance network info cache due to event network-changed-12c430b0-3ca9-4fe0-ad3d-30068930d0e7. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 861.500485] env[64020]: DEBUG oslo_concurrency.lockutils [req-a7e4de68-d64c-48ba-bb08-6d86f8950593 req-eb04ac33-ecb8-47a3-8069-df48e3277328 service nova] Acquiring lock "refresh_cache-dcb3a0b0-a558-45be-8208-038e465dc802" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 861.500766] env[64020]: DEBUG oslo_concurrency.lockutils [req-a7e4de68-d64c-48ba-bb08-6d86f8950593 req-eb04ac33-ecb8-47a3-8069-df48e3277328 service nova] Acquired lock "refresh_cache-dcb3a0b0-a558-45be-8208-038e465dc802" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 861.501074] env[64020]: DEBUG nova.network.neutron [req-a7e4de68-d64c-48ba-bb08-6d86f8950593 req-eb04ac33-ecb8-47a3-8069-df48e3277328 service nova] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Refreshing network info cache for port 12c430b0-3ca9-4fe0-ad3d-30068930d0e7 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 861.501651] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-a7e4de68-d64c-48ba-bb08-6d86f8950593 req-eb04ac33-ecb8-47a3-8069-df48e3277328 service nova] Expecting reply to msg f113ef1c07af458db1ba4c9a1a26e10b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 861.510983] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f113ef1c07af458db1ba4c9a1a26e10b [ 861.512516] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e79dd2b5-c023-4b6e-97ae-f84a207985b3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.606609] env[64020]: ERROR nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 12c430b0-3ca9-4fe0-ad3d-30068930d0e7, please check neutron logs for more information. [ 861.606609] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 861.606609] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 861.606609] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 861.606609] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 861.606609] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 861.606609] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 861.606609] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 861.606609] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 861.606609] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 861.606609] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 861.606609] env[64020]: ERROR nova.compute.manager raise self.value [ 861.606609] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 861.606609] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 861.606609] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 861.606609] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 861.607078] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 861.607078] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 861.607078] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 12c430b0-3ca9-4fe0-ad3d-30068930d0e7, please check neutron logs for more information. [ 861.607078] env[64020]: ERROR nova.compute.manager [ 861.607495] env[64020]: Traceback (most recent call last): [ 861.607614] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 861.607614] env[64020]: listener.cb(fileno) [ 861.607699] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 861.607699] env[64020]: result = function(*args, **kwargs) [ 861.607784] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 861.607784] env[64020]: return func(*args, **kwargs) [ 861.607868] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 861.607868] env[64020]: raise e [ 861.607942] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 861.607942] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 861.608032] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 861.608032] env[64020]: created_port_ids = self._update_ports_for_instance( [ 861.608109] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 861.608109] env[64020]: with excutils.save_and_reraise_exception(): [ 861.608197] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 861.608197] env[64020]: self.force_reraise() [ 861.608279] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 861.608279] env[64020]: raise self.value [ 861.608353] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 861.608353] env[64020]: updated_port = self._update_port( [ 861.608425] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 861.608425] env[64020]: _ensure_no_port_binding_failure(port) [ 861.608497] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 861.608497] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 861.608587] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 12c430b0-3ca9-4fe0-ad3d-30068930d0e7, please check neutron logs for more information. [ 861.608647] env[64020]: Removing descriptor: 16 [ 861.609456] env[64020]: ERROR nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 12c430b0-3ca9-4fe0-ad3d-30068930d0e7, please check neutron logs for more information. [ 861.609456] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Traceback (most recent call last): [ 861.609456] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 861.609456] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] yield resources [ 861.609456] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 861.609456] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] self.driver.spawn(context, instance, image_meta, [ 861.609456] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 861.609456] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] self._vmops.spawn(context, instance, image_meta, injected_files, [ 861.609456] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 861.609456] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] vm_ref = self.build_virtual_machine(instance, [ 861.609456] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] vif_infos = vmwarevif.get_vif_info(self._session, [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] for vif in network_info: [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] return self._sync_wrapper(fn, *args, **kwargs) [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] self.wait() [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] self[:] = self._gt.wait() [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] return self._exit_event.wait() [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 861.609755] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] result = hub.switch() [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] return self.greenlet.switch() [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] result = function(*args, **kwargs) [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] return func(*args, **kwargs) [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] raise e [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] nwinfo = self.network_api.allocate_for_instance( [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] created_port_ids = self._update_ports_for_instance( [ 861.610106] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] with excutils.save_and_reraise_exception(): [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] self.force_reraise() [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] raise self.value [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] updated_port = self._update_port( [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] _ensure_no_port_binding_failure(port) [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] raise exception.PortBindingFailed(port_id=port['id']) [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] nova.exception.PortBindingFailed: Binding failed for port 12c430b0-3ca9-4fe0-ad3d-30068930d0e7, please check neutron logs for more information. [ 861.610400] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] [ 861.611257] env[64020]: INFO nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Terminating instance [ 861.617023] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "refresh_cache-dcb3a0b0-a558-45be-8208-038e465dc802" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 861.715016] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77473df4-d413-4a5e-8484-98444e04bb14 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.722526] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14cf5b67-3937-4bdf-9e33-31ee52ce348c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.751803] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e34945c5-f2c9-4d99-86a4-1313cb30ef48 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.760399] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d1a8832-13ce-4701-946c-9aaa29df6584 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.773333] env[64020]: DEBUG nova.compute.provider_tree [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 861.774019] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 2b0eba74d1ed43098b38b76d791c726a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 861.781143] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b0eba74d1ed43098b38b76d791c726a [ 861.958491] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 861.959204] env[64020]: DEBUG nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 861.959509] env[64020]: DEBUG nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 861.959867] env[64020]: DEBUG nova.network.neutron [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 861.973743] env[64020]: DEBUG nova.network.neutron [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 861.974400] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg bf328806a10e4d7ba7b9f6e2e8df2c62 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 861.981514] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf328806a10e4d7ba7b9f6e2e8df2c62 [ 862.022957] env[64020]: DEBUG nova.network.neutron [req-a7e4de68-d64c-48ba-bb08-6d86f8950593 req-eb04ac33-ecb8-47a3-8069-df48e3277328 service nova] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 862.118051] env[64020]: DEBUG nova.network.neutron [req-a7e4de68-d64c-48ba-bb08-6d86f8950593 req-eb04ac33-ecb8-47a3-8069-df48e3277328 service nova] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 862.118728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-a7e4de68-d64c-48ba-bb08-6d86f8950593 req-eb04ac33-ecb8-47a3-8069-df48e3277328 service nova] Expecting reply to msg 18817df0308b46e2b9b62fcb94e6e1b7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 862.131964] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18817df0308b46e2b9b62fcb94e6e1b7 [ 862.280227] env[64020]: DEBUG nova.scheduler.client.report [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 862.280227] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 5ce71e2203d24ad58d476ae06636a7b6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 862.291574] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ce71e2203d24ad58d476ae06636a7b6 [ 862.443951] env[64020]: DEBUG nova.compute.manager [None req-43f9ba8c-2330-4f65-9e39-98f832048ddb tempest-ServerDiagnosticsV248Test-610054911 tempest-ServerDiagnosticsV248Test-610054911-project-admin] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 862.445243] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6627ca1b-ad6d-4fa1-92c3-6d95c5878d9f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.451993] env[64020]: INFO nova.compute.manager [None req-43f9ba8c-2330-4f65-9e39-98f832048ddb tempest-ServerDiagnosticsV248Test-610054911 tempest-ServerDiagnosticsV248Test-610054911-project-admin] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Retrieving diagnostics [ 862.452819] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-466cd2ef-e670-4a60-a0a5-abc6e6b36b22 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.483009] env[64020]: DEBUG nova.network.neutron [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 862.483682] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg bedf90f8cb9c413684b5950070f6b293 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 862.493451] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bedf90f8cb9c413684b5950070f6b293 [ 862.621468] env[64020]: DEBUG oslo_concurrency.lockutils [req-a7e4de68-d64c-48ba-bb08-6d86f8950593 req-eb04ac33-ecb8-47a3-8069-df48e3277328 service nova] Releasing lock "refresh_cache-dcb3a0b0-a558-45be-8208-038e465dc802" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 862.622078] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquired lock "refresh_cache-dcb3a0b0-a558-45be-8208-038e465dc802" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 862.622435] env[64020]: DEBUG nova.network.neutron [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 862.623006] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg cb08fa5d6ab345ea989b1236296b62bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 862.631514] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cb08fa5d6ab345ea989b1236296b62bc [ 862.782230] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.949s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 862.784737] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 31.357s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 862.786939] env[64020]: INFO nova.compute.claims [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 862.788647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 57d274550b5440819f1d9fa8ad36a116 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 862.806559] env[64020]: INFO nova.scheduler.client.report [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Deleted allocations for instance e27f988a-a64b-407b-ad66-4bed7ee45b52 [ 862.822434] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg 37f8c9b258a94a68bbef35ca35485c6a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 862.829071] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 57d274550b5440819f1d9fa8ad36a116 [ 862.865717] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37f8c9b258a94a68bbef35ca35485c6a [ 862.987411] env[64020]: INFO nova.compute.manager [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3] Took 1.03 seconds to deallocate network for instance. [ 862.989242] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 86958af07ac049d589c251dce7d9f373 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 863.010552] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg be4dfc76e5c54ebb9d95bc85614cb4cb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 863.019118] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86958af07ac049d589c251dce7d9f373 [ 863.019923] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be4dfc76e5c54ebb9d95bc85614cb4cb [ 863.142212] env[64020]: DEBUG nova.network.neutron [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 863.259208] env[64020]: DEBUG nova.network.neutron [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 863.259755] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 6de4b17fb4704b7db0fff541cff8b4e7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 863.267639] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6de4b17fb4704b7db0fff541cff8b4e7 [ 863.292304] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg d95d0bde189c4095a9da7542e6ff6515 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 863.299101] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d95d0bde189c4095a9da7542e6ff6515 [ 863.327155] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Lock "e27f988a-a64b-407b-ad66-4bed7ee45b52" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 36.511s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 863.327481] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7ff5819c-7417-48b9-9e0e-c2cd1a102273 tempest-ServerShowV254Test-1080939031 tempest-ServerShowV254Test-1080939031-project-member] Expecting reply to msg fa1d3ebe73734ff4bee35c1667c3c1df in queue reply_57893177120949e6a93cb88e15cd42b4 [ 863.337696] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fa1d3ebe73734ff4bee35c1667c3c1df [ 863.493837] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 4d04ee556c874481acd86c59c5d302ed in queue reply_57893177120949e6a93cb88e15cd42b4 [ 863.512625] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquiring lock "995288b5-d79a-4af3-a1e8-3571fff2d356" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 863.512874] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Lock "995288b5-d79a-4af3-a1e8-3571fff2d356" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 863.513080] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquiring lock "995288b5-d79a-4af3-a1e8-3571fff2d356-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 863.513651] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Lock "995288b5-d79a-4af3-a1e8-3571fff2d356-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 863.513834] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Lock "995288b5-d79a-4af3-a1e8-3571fff2d356-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 863.516424] env[64020]: INFO nova.compute.manager [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Terminating instance [ 863.519373] env[64020]: DEBUG nova.compute.manager [req-373ac214-cedc-4e73-baaa-7feab2bb5892 req-f130f8a3-7ef5-4100-9732-0ec2572acd31 service nova] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Received event network-vif-deleted-12c430b0-3ca9-4fe0-ad3d-30068930d0e7 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 863.520218] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquiring lock "refresh_cache-995288b5-d79a-4af3-a1e8-3571fff2d356" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 863.520418] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquired lock "refresh_cache-995288b5-d79a-4af3-a1e8-3571fff2d356" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 863.520615] env[64020]: DEBUG nova.network.neutron [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 863.521239] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 0547539da8664747b54aba05fc1b624d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 863.527923] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0547539da8664747b54aba05fc1b624d [ 863.530225] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d04ee556c874481acd86c59c5d302ed [ 863.762609] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Releasing lock "refresh_cache-dcb3a0b0-a558-45be-8208-038e465dc802" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 863.763039] env[64020]: DEBUG nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 863.763225] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 863.763528] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-840bdc19-dab9-47c5-bda5-c48be00cc96a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 863.772545] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f99a194c-e375-41ce-a0c7-0ff6b02efd80 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 863.796622] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance dcb3a0b0-a558-45be-8208-038e465dc802 could not be found. [ 863.796823] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 863.796993] env[64020]: INFO nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Took 0.03 seconds to destroy the instance on the hypervisor. [ 863.797218] env[64020]: DEBUG oslo.service.loopingcall [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 863.797596] env[64020]: DEBUG nova.compute.manager [-] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 863.797690] env[64020]: DEBUG nova.network.neutron [-] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 863.811459] env[64020]: DEBUG nova.network.neutron [-] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 863.811978] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3352534470d147eb9f7dad607b6aea8f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 863.819273] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3352534470d147eb9f7dad607b6aea8f [ 864.023790] env[64020]: INFO nova.scheduler.client.report [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Deleted allocations for instance 9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3 [ 864.033883] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 35b3684f354546288f5f7a6d1d5ad198 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 864.044051] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35b3684f354546288f5f7a6d1d5ad198 [ 864.052973] env[64020]: DEBUG nova.network.neutron [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 864.106877] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-443aa7ca-53d6-4ac3-a61d-65c9c6e45a9f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.115076] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e856d3c2-1994-4204-ba51-75f85bb372db {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.146107] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4af42288-4715-4490-b8fa-66eaeb28c093 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.153204] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a5191df-d040-4f80-b93e-ac175ee8105a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.165856] env[64020]: DEBUG nova.compute.provider_tree [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 864.166416] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 5dd2e0f25189427191cf586c5a81d396 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 864.174213] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5dd2e0f25189427191cf586c5a81d396 [ 864.186026] env[64020]: DEBUG nova.network.neutron [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 864.186500] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg b316ba1f6c3d498ba8df72d5a7b6cd97 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 864.195303] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b316ba1f6c3d498ba8df72d5a7b6cd97 [ 864.313870] env[64020]: DEBUG nova.network.neutron [-] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 864.314338] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b3f38cc87da045c487a4525b5f70dc5f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 864.322982] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3f38cc87da045c487a4525b5f70dc5f [ 864.536525] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa1bc611-a5fe-4716-80d7-6afed437df59 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "9684ca7d-6a74-4ef9-8dfe-64b0987b6bb3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 176.407s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.537094] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg d44506a35d064f9ba42616528f796444 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 864.547424] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d44506a35d064f9ba42616528f796444 [ 864.669463] env[64020]: DEBUG nova.scheduler.client.report [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 864.671890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 98709db9b66147b980a74963c385c678 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 864.685598] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98709db9b66147b980a74963c385c678 [ 864.688574] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Releasing lock "refresh_cache-995288b5-d79a-4af3-a1e8-3571fff2d356" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 864.688952] env[64020]: DEBUG nova.compute.manager [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 864.689150] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 864.690204] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c85a7835-4c88-44d0-babe-98808f9d7c44 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.698166] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 864.698384] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-1012ceb3-ee79-46dc-8942-989f089c108a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.704082] env[64020]: DEBUG oslo_vmware.api [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for the task: (returnval){ [ 864.704082] env[64020]: value = "task-407902" [ 864.704082] env[64020]: _type = "Task" [ 864.704082] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 864.711911] env[64020]: DEBUG oslo_vmware.api [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407902, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 864.816907] env[64020]: INFO nova.compute.manager [-] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Took 1.02 seconds to deallocate network for instance. [ 864.819250] env[64020]: DEBUG nova.compute.claims [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 864.819418] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 865.039094] env[64020]: DEBUG nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 865.040956] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 7571fa56e42c45808634870d0917f5c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 865.071268] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7571fa56e42c45808634870d0917f5c8 [ 865.174558] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.390s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 865.175092] env[64020]: DEBUG nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 865.177273] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 005cba79c6a04fb18af28148f702c7bd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 865.178334] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 32.120s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 865.179713] env[64020]: INFO nova.compute.claims [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 865.181189] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 19278025535240279b723224d0d44012 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 865.214227] env[64020]: DEBUG oslo_vmware.api [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407902, 'name': PowerOffVM_Task, 'duration_secs': 0.107466} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 865.214504] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 865.214661] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 865.215284] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19278025535240279b723224d0d44012 [ 865.215658] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-5af5a128-7649-4211-afc9-086e4b5dd19d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 865.221686] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 005cba79c6a04fb18af28148f702c7bd [ 865.241776] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 865.241980] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Deleting contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 865.242210] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Deleting the datastore file [datastore1] 995288b5-d79a-4af3-a1e8-3571fff2d356 {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 865.242514] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-0848c9c5-b2cd-4fd6-921c-685e81d93905 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 865.248514] env[64020]: DEBUG oslo_vmware.api [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for the task: (returnval){ [ 865.248514] env[64020]: value = "task-407904" [ 865.248514] env[64020]: _type = "Task" [ 865.248514] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 865.260343] env[64020]: DEBUG oslo_vmware.api [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407904, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 865.560682] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 865.684556] env[64020]: DEBUG nova.compute.utils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 865.684838] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 76de73cedbd04e56a6fbf562b080c684 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 865.686856] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 66621884f89a4257be61019623dbeb9b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 865.687848] env[64020]: DEBUG nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 865.687923] env[64020]: DEBUG nova.network.neutron [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 865.698343] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66621884f89a4257be61019623dbeb9b [ 865.699901] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 76de73cedbd04e56a6fbf562b080c684 [ 865.748129] env[64020]: DEBUG nova.policy [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '0728aba0e9f64ee585afda72e5fee1ca', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '475308e79f7143358e0d0315f13887c3', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 865.759634] env[64020]: DEBUG oslo_vmware.api [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Task: {'id': task-407904, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.08721} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 865.759858] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 865.760039] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Deleted contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 865.760206] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 865.760371] env[64020]: INFO nova.compute.manager [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Took 1.07 seconds to destroy the instance on the hypervisor. [ 865.760595] env[64020]: DEBUG oslo.service.loopingcall [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 865.760768] env[64020]: DEBUG nova.compute.manager [-] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 865.760886] env[64020]: DEBUG nova.network.neutron [-] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 865.780382] env[64020]: DEBUG nova.network.neutron [-] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 865.781132] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bddf886bd7984e65bfcd594153bbd11e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 865.789103] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bddf886bd7984e65bfcd594153bbd11e [ 866.107569] env[64020]: DEBUG nova.network.neutron [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Successfully created port: 73b13abf-97df-4e7d-9625-2864f0a646e9 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 866.188442] env[64020]: DEBUG nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 866.190649] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg c2ec6de302434c2e9b90c4c569a26a55 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 866.247011] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2ec6de302434c2e9b90c4c569a26a55 [ 866.286603] env[64020]: DEBUG nova.network.neutron [-] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 866.287047] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8f1742a3e1244fad9ea3ff278f338c0e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 866.296430] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f1742a3e1244fad9ea3ff278f338c0e [ 866.323231] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.323458] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.493516] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6997fdcb-cd3a-4523-b2fe-4f5e37f6abbc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.501288] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5116afd-57a1-4673-b254-d71cca4bbcb7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.530938] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aff15450-e5d7-4ac7-aae2-5c4f914c055c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.537647] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4f46bb6-ad2d-4de7-af11-1eefe8d4a0f6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.551967] env[64020]: DEBUG nova.compute.provider_tree [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 866.552389] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg a842a7175181405cb39adfb6452eb4f2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 866.559932] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a842a7175181405cb39adfb6452eb4f2 [ 866.697647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg bf876177be874a91bf6017ffec890f51 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 866.728380] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf876177be874a91bf6017ffec890f51 [ 866.796060] env[64020]: INFO nova.compute.manager [-] [instance: 995288b5-d79a-4af3-a1e8-3571fff2d356] Took 1.04 seconds to deallocate network for instance. [ 866.799814] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 975affc566364b8498a38d78c952545b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 866.826923] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 975affc566364b8498a38d78c952545b [ 866.944502] env[64020]: DEBUG nova.compute.manager [req-953e8de9-cb13-4ea2-afdf-ed64bcd424df req-48e5987f-8359-4e72-9c93-84bdb30373a3 service nova] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Received event network-changed-73b13abf-97df-4e7d-9625-2864f0a646e9 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 866.944679] env[64020]: DEBUG nova.compute.manager [req-953e8de9-cb13-4ea2-afdf-ed64bcd424df req-48e5987f-8359-4e72-9c93-84bdb30373a3 service nova] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Refreshing instance network info cache due to event network-changed-73b13abf-97df-4e7d-9625-2864f0a646e9. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 866.944887] env[64020]: DEBUG oslo_concurrency.lockutils [req-953e8de9-cb13-4ea2-afdf-ed64bcd424df req-48e5987f-8359-4e72-9c93-84bdb30373a3 service nova] Acquiring lock "refresh_cache-c9a9eccd-4730-4815-8270-8272ac8bdec6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 866.945019] env[64020]: DEBUG oslo_concurrency.lockutils [req-953e8de9-cb13-4ea2-afdf-ed64bcd424df req-48e5987f-8359-4e72-9c93-84bdb30373a3 service nova] Acquired lock "refresh_cache-c9a9eccd-4730-4815-8270-8272ac8bdec6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 866.945178] env[64020]: DEBUG nova.network.neutron [req-953e8de9-cb13-4ea2-afdf-ed64bcd424df req-48e5987f-8359-4e72-9c93-84bdb30373a3 service nova] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Refreshing network info cache for port 73b13abf-97df-4e7d-9625-2864f0a646e9 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 866.945585] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-953e8de9-cb13-4ea2-afdf-ed64bcd424df req-48e5987f-8359-4e72-9c93-84bdb30373a3 service nova] Expecting reply to msg 6b63305cda7d4fdda718e6dda5c554d6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 866.952520] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6b63305cda7d4fdda718e6dda5c554d6 [ 867.055029] env[64020]: DEBUG nova.scheduler.client.report [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 867.057625] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 71229b7b1e5a41d6a0dd4142f48435bd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 867.071164] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 71229b7b1e5a41d6a0dd4142f48435bd [ 867.211691] env[64020]: DEBUG nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 867.230487] env[64020]: ERROR nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 73b13abf-97df-4e7d-9625-2864f0a646e9, please check neutron logs for more information. [ 867.230487] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 867.230487] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 867.230487] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 867.230487] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 867.230487] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 867.230487] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 867.230487] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 867.230487] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 867.230487] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 867.230487] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 867.230487] env[64020]: ERROR nova.compute.manager raise self.value [ 867.230487] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 867.230487] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 867.230487] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 867.230487] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 867.230920] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 867.230920] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 867.230920] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 73b13abf-97df-4e7d-9625-2864f0a646e9, please check neutron logs for more information. [ 867.230920] env[64020]: ERROR nova.compute.manager [ 867.230920] env[64020]: Traceback (most recent call last): [ 867.230920] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 867.230920] env[64020]: listener.cb(fileno) [ 867.230920] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 867.230920] env[64020]: result = function(*args, **kwargs) [ 867.230920] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 867.230920] env[64020]: return func(*args, **kwargs) [ 867.230920] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 867.230920] env[64020]: raise e [ 867.230920] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 867.230920] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 867.230920] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 867.230920] env[64020]: created_port_ids = self._update_ports_for_instance( [ 867.230920] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 867.230920] env[64020]: with excutils.save_and_reraise_exception(): [ 867.230920] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 867.230920] env[64020]: self.force_reraise() [ 867.230920] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 867.230920] env[64020]: raise self.value [ 867.230920] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 867.230920] env[64020]: updated_port = self._update_port( [ 867.230920] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 867.230920] env[64020]: _ensure_no_port_binding_failure(port) [ 867.230920] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 867.230920] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 867.231725] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 73b13abf-97df-4e7d-9625-2864f0a646e9, please check neutron logs for more information. [ 867.231725] env[64020]: Removing descriptor: 16 [ 867.242687] env[64020]: DEBUG nova.virt.hardware [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 867.242925] env[64020]: DEBUG nova.virt.hardware [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 867.243074] env[64020]: DEBUG nova.virt.hardware [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 867.243251] env[64020]: DEBUG nova.virt.hardware [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 867.243456] env[64020]: DEBUG nova.virt.hardware [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 867.243630] env[64020]: DEBUG nova.virt.hardware [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 867.243840] env[64020]: DEBUG nova.virt.hardware [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 867.243995] env[64020]: DEBUG nova.virt.hardware [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 867.244212] env[64020]: DEBUG nova.virt.hardware [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 867.244379] env[64020]: DEBUG nova.virt.hardware [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 867.244546] env[64020]: DEBUG nova.virt.hardware [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 867.245396] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-138dbb56-3564-4b99-a468-a492115d00d6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.253378] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2abd9116-f3f2-40e4-931d-099a058ab414 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.269367] env[64020]: ERROR nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 73b13abf-97df-4e7d-9625-2864f0a646e9, please check neutron logs for more information. [ 867.269367] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Traceback (most recent call last): [ 867.269367] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 867.269367] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] yield resources [ 867.269367] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 867.269367] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] self.driver.spawn(context, instance, image_meta, [ 867.269367] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 867.269367] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 867.269367] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 867.269367] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] vm_ref = self.build_virtual_machine(instance, [ 867.269367] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] vif_infos = vmwarevif.get_vif_info(self._session, [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] for vif in network_info: [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] return self._sync_wrapper(fn, *args, **kwargs) [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] self.wait() [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] self[:] = self._gt.wait() [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] return self._exit_event.wait() [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 867.269966] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] current.throw(*self._exc) [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] result = function(*args, **kwargs) [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] return func(*args, **kwargs) [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] raise e [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] nwinfo = self.network_api.allocate_for_instance( [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] created_port_ids = self._update_ports_for_instance( [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] with excutils.save_and_reraise_exception(): [ 867.270521] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 867.271103] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] self.force_reraise() [ 867.271103] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 867.271103] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] raise self.value [ 867.271103] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 867.271103] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] updated_port = self._update_port( [ 867.271103] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 867.271103] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] _ensure_no_port_binding_failure(port) [ 867.271103] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 867.271103] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] raise exception.PortBindingFailed(port_id=port['id']) [ 867.271103] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] nova.exception.PortBindingFailed: Binding failed for port 73b13abf-97df-4e7d-9625-2864f0a646e9, please check neutron logs for more information. [ 867.271103] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] [ 867.271103] env[64020]: INFO nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Terminating instance [ 867.271944] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Acquiring lock "refresh_cache-c9a9eccd-4730-4815-8270-8272ac8bdec6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 867.303272] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 867.463172] env[64020]: DEBUG nova.network.neutron [req-953e8de9-cb13-4ea2-afdf-ed64bcd424df req-48e5987f-8359-4e72-9c93-84bdb30373a3 service nova] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 867.549746] env[64020]: DEBUG nova.network.neutron [req-953e8de9-cb13-4ea2-afdf-ed64bcd424df req-48e5987f-8359-4e72-9c93-84bdb30373a3 service nova] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 867.550251] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-953e8de9-cb13-4ea2-afdf-ed64bcd424df req-48e5987f-8359-4e72-9c93-84bdb30373a3 service nova] Expecting reply to msg 05135bb6765c40b9b94b98b740598116 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 867.558334] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05135bb6765c40b9b94b98b740598116 [ 867.560492] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.382s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 867.560964] env[64020]: DEBUG nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 867.562671] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg eb5d410f054441c7a5a6f7cea49144f9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 867.563627] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.896s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 867.565305] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg e829d788ca78459e9651f9609bf3e09b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 867.596875] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb5d410f054441c7a5a6f7cea49144f9 [ 867.596875] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e829d788ca78459e9651f9609bf3e09b [ 868.052265] env[64020]: DEBUG oslo_concurrency.lockutils [req-953e8de9-cb13-4ea2-afdf-ed64bcd424df req-48e5987f-8359-4e72-9c93-84bdb30373a3 service nova] Releasing lock "refresh_cache-c9a9eccd-4730-4815-8270-8272ac8bdec6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 868.052703] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Acquired lock "refresh_cache-c9a9eccd-4730-4815-8270-8272ac8bdec6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 868.052913] env[64020]: DEBUG nova.network.neutron [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 868.053361] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 2d04e917da444dc49367294f485761e7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 868.061157] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d04e917da444dc49367294f485761e7 [ 868.068562] env[64020]: DEBUG nova.compute.utils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 868.069174] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 2aaf7d9872704f249dabd6c13b182b60 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 868.076733] env[64020]: DEBUG nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 868.076889] env[64020]: DEBUG nova.network.neutron [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 868.085447] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2aaf7d9872704f249dabd6c13b182b60 [ 868.152391] env[64020]: DEBUG nova.policy [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b3c86a06615e4c988cce3e6ac5dfc3b3', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'b3a58e3eeb554aedb1b1c06ef73ea46d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 868.387796] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8553d802-9ff8-4ddf-a233-0652e2f22f4a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.396163] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1daa0a7d-da08-4eb2-969d-be16f46000c7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.428491] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-006b770d-5792-4c1c-88f4-59fafc60f00d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.438737] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64215f35-43ee-4aea-aee7-313eaac69a88 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.453033] env[64020]: DEBUG nova.compute.provider_tree [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 868.454014] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 882fab65aedc4043a11c88f75b01519e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 868.461236] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 882fab65aedc4043a11c88f75b01519e [ 868.538526] env[64020]: DEBUG nova.network.neutron [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Successfully created port: 842426d0-e222-405d-a943-e75a3af7e696 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 868.578353] env[64020]: DEBUG nova.network.neutron [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 868.580783] env[64020]: DEBUG nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 868.583152] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg cb782f8aeec840abbfd4904fdb19da77 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 868.622695] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cb782f8aeec840abbfd4904fdb19da77 [ 868.668746] env[64020]: DEBUG nova.network.neutron [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 868.669366] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg e184379dc3904936a626e756af943519 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 868.678140] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e184379dc3904936a626e756af943519 [ 868.957163] env[64020]: DEBUG nova.scheduler.client.report [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 868.959718] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 1f5b426ce1094bbdb58a5bc1b6836a6e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 868.973631] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f5b426ce1094bbdb58a5bc1b6836a6e [ 869.101388] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 4694d9c10faf4b1e9f305d9b336fabbc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 869.137195] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4694d9c10faf4b1e9f305d9b336fabbc [ 869.172512] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Releasing lock "refresh_cache-c9a9eccd-4730-4815-8270-8272ac8bdec6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 869.173000] env[64020]: DEBUG nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 869.173334] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 869.173496] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1eb4c9c6-57ad-4cca-8584-31ad05268b65 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.184033] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1046bdc0-de29-4ee1-aaea-f70a9ffc9f6f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.196812] env[64020]: DEBUG nova.compute.manager [req-4082ee1e-b617-4279-9f2a-bf1385b6de60 req-14616c7f-7f00-4c81-b73c-59361dc7843f service nova] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Received event network-vif-deleted-73b13abf-97df-4e7d-9625-2864f0a646e9 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 869.214890] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c9a9eccd-4730-4815-8270-8272ac8bdec6 could not be found. [ 869.215109] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 869.215277] env[64020]: INFO nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Took 0.04 seconds to destroy the instance on the hypervisor. [ 869.215509] env[64020]: DEBUG oslo.service.loopingcall [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 869.215769] env[64020]: DEBUG nova.compute.manager [-] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 869.215830] env[64020]: DEBUG nova.network.neutron [-] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 869.230461] env[64020]: DEBUG nova.network.neutron [-] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 869.231250] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 9e3f028e274e4aab9aa8700775d55c4e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 869.238006] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e3f028e274e4aab9aa8700775d55c4e [ 869.462402] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.899s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 869.463034] env[64020]: ERROR nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3dce5495-91e7-461f-a6e6-b9be33af5bf0, please check neutron logs for more information. [ 869.463034] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Traceback (most recent call last): [ 869.463034] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 869.463034] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] self.driver.spawn(context, instance, image_meta, [ 869.463034] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 869.463034] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 869.463034] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 869.463034] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] vm_ref = self.build_virtual_machine(instance, [ 869.463034] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 869.463034] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] vif_infos = vmwarevif.get_vif_info(self._session, [ 869.463034] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] for vif in network_info: [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] return self._sync_wrapper(fn, *args, **kwargs) [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] self.wait() [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] self[:] = self._gt.wait() [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] return self._exit_event.wait() [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] current.throw(*self._exc) [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 869.463366] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] result = function(*args, **kwargs) [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] return func(*args, **kwargs) [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] raise e [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] nwinfo = self.network_api.allocate_for_instance( [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] created_port_ids = self._update_ports_for_instance( [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] with excutils.save_and_reraise_exception(): [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] self.force_reraise() [ 869.463716] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 869.464107] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] raise self.value [ 869.464107] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 869.464107] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] updated_port = self._update_port( [ 869.464107] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 869.464107] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] _ensure_no_port_binding_failure(port) [ 869.464107] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 869.464107] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] raise exception.PortBindingFailed(port_id=port['id']) [ 869.464107] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] nova.exception.PortBindingFailed: Binding failed for port 3dce5495-91e7-461f-a6e6-b9be33af5bf0, please check neutron logs for more information. [ 869.464107] env[64020]: ERROR nova.compute.manager [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] [ 869.464107] env[64020]: DEBUG nova.compute.utils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Binding failed for port 3dce5495-91e7-461f-a6e6-b9be33af5bf0, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 869.464947] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.234s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 869.466440] env[64020]: INFO nova.compute.claims [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 869.468135] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg fdee176ce4fb413cb5a262ef1ab06733 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 869.469275] env[64020]: DEBUG nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Build of instance 45167852-b7c0-4614-89f1-f8f7fc2078f2 was re-scheduled: Binding failed for port 3dce5495-91e7-461f-a6e6-b9be33af5bf0, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 869.469680] env[64020]: DEBUG nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 869.469893] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Acquiring lock "refresh_cache-45167852-b7c0-4614-89f1-f8f7fc2078f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 869.470031] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Acquired lock "refresh_cache-45167852-b7c0-4614-89f1-f8f7fc2078f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 869.470319] env[64020]: DEBUG nova.network.neutron [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 869.470533] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg f3df70c1f8224e5c830d3d9420642c0a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 869.477001] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3df70c1f8224e5c830d3d9420642c0a [ 869.500633] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fdee176ce4fb413cb5a262ef1ab06733 [ 869.605555] env[64020]: DEBUG nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 869.632450] env[64020]: DEBUG nova.virt.hardware [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 869.632764] env[64020]: DEBUG nova.virt.hardware [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 869.632899] env[64020]: DEBUG nova.virt.hardware [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 869.633070] env[64020]: DEBUG nova.virt.hardware [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 869.633205] env[64020]: DEBUG nova.virt.hardware [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 869.633341] env[64020]: DEBUG nova.virt.hardware [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 869.633537] env[64020]: DEBUG nova.virt.hardware [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 869.633686] env[64020]: DEBUG nova.virt.hardware [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 869.633841] env[64020]: DEBUG nova.virt.hardware [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 869.634010] env[64020]: DEBUG nova.virt.hardware [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 869.634224] env[64020]: DEBUG nova.virt.hardware [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 869.635649] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91830835-18c5-4872-a736-21b54d8e916b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.644373] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73ef10ba-8521-43e2-b38f-62de5c31dda1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.733297] env[64020]: DEBUG nova.network.neutron [-] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 869.733726] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 461bbde1d0a04ba28912efa6db8a90cc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 869.742130] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 461bbde1d0a04ba28912efa6db8a90cc [ 869.833368] env[64020]: ERROR nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 842426d0-e222-405d-a943-e75a3af7e696, please check neutron logs for more information. [ 869.833368] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 869.833368] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 869.833368] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 869.833368] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 869.833368] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 869.833368] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 869.833368] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 869.833368] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 869.833368] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 869.833368] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 869.833368] env[64020]: ERROR nova.compute.manager raise self.value [ 869.833368] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 869.833368] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 869.833368] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 869.833368] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 869.833888] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 869.833888] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 869.833888] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 842426d0-e222-405d-a943-e75a3af7e696, please check neutron logs for more information. [ 869.833888] env[64020]: ERROR nova.compute.manager [ 869.833888] env[64020]: Traceback (most recent call last): [ 869.833888] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 869.833888] env[64020]: listener.cb(fileno) [ 869.833888] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 869.833888] env[64020]: result = function(*args, **kwargs) [ 869.833888] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 869.833888] env[64020]: return func(*args, **kwargs) [ 869.833888] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 869.833888] env[64020]: raise e [ 869.833888] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 869.833888] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 869.833888] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 869.833888] env[64020]: created_port_ids = self._update_ports_for_instance( [ 869.833888] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 869.833888] env[64020]: with excutils.save_and_reraise_exception(): [ 869.833888] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 869.833888] env[64020]: self.force_reraise() [ 869.833888] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 869.833888] env[64020]: raise self.value [ 869.833888] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 869.833888] env[64020]: updated_port = self._update_port( [ 869.833888] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 869.833888] env[64020]: _ensure_no_port_binding_failure(port) [ 869.833888] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 869.833888] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 869.834831] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 842426d0-e222-405d-a943-e75a3af7e696, please check neutron logs for more information. [ 869.834831] env[64020]: Removing descriptor: 16 [ 869.834831] env[64020]: ERROR nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 842426d0-e222-405d-a943-e75a3af7e696, please check neutron logs for more information. [ 869.834831] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Traceback (most recent call last): [ 869.834831] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 869.834831] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] yield resources [ 869.834831] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 869.834831] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] self.driver.spawn(context, instance, image_meta, [ 869.834831] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 869.834831] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] self._vmops.spawn(context, instance, image_meta, injected_files, [ 869.834831] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 869.834831] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] vm_ref = self.build_virtual_machine(instance, [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] vif_infos = vmwarevif.get_vif_info(self._session, [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] for vif in network_info: [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] return self._sync_wrapper(fn, *args, **kwargs) [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] self.wait() [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] self[:] = self._gt.wait() [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] return self._exit_event.wait() [ 869.835156] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] result = hub.switch() [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] return self.greenlet.switch() [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] result = function(*args, **kwargs) [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] return func(*args, **kwargs) [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] raise e [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] nwinfo = self.network_api.allocate_for_instance( [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 869.835500] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] created_port_ids = self._update_ports_for_instance( [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] with excutils.save_and_reraise_exception(): [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] self.force_reraise() [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] raise self.value [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] updated_port = self._update_port( [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] _ensure_no_port_binding_failure(port) [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 869.835820] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] raise exception.PortBindingFailed(port_id=port['id']) [ 869.836162] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] nova.exception.PortBindingFailed: Binding failed for port 842426d0-e222-405d-a943-e75a3af7e696, please check neutron logs for more information. [ 869.836162] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] [ 869.836162] env[64020]: INFO nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Terminating instance [ 869.836548] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Acquiring lock "refresh_cache-f4682b82-7475-412b-9319-bee1f05a9c63" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 869.836706] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Acquired lock "refresh_cache-f4682b82-7475-412b-9319-bee1f05a9c63" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 869.836871] env[64020]: DEBUG nova.network.neutron [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 869.837281] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg bab1e480301843cb8a21f25c2049e02f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 869.843794] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bab1e480301843cb8a21f25c2049e02f [ 869.976192] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg ba484aade68f475c823529861749734f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 869.985235] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba484aade68f475c823529861749734f [ 869.990865] env[64020]: DEBUG nova.network.neutron [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 870.064692] env[64020]: DEBUG nova.network.neutron [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 870.065194] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 9967f25e2bca46059ad455dcee446a78 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 870.073960] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9967f25e2bca46059ad455dcee446a78 [ 870.236345] env[64020]: INFO nova.compute.manager [-] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Took 1.02 seconds to deallocate network for instance. [ 870.238663] env[64020]: DEBUG nova.compute.claims [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 870.238827] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 870.354254] env[64020]: DEBUG nova.network.neutron [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 870.417342] env[64020]: DEBUG nova.network.neutron [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 870.417342] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg e08a9d217d51493698d1d7221beb3817 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 870.426225] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e08a9d217d51493698d1d7221beb3817 [ 870.567049] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Releasing lock "refresh_cache-45167852-b7c0-4614-89f1-f8f7fc2078f2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 870.567329] env[64020]: DEBUG nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 870.567635] env[64020]: DEBUG nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 870.567889] env[64020]: DEBUG nova.network.neutron [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 870.585535] env[64020]: DEBUG nova.network.neutron [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 870.586068] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 26ee7f3e9c624445858901cfece7792c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 870.592857] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26ee7f3e9c624445858901cfece7792c [ 870.766419] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a52c26fd-bd0c-4ea5-8564-8192a4d5f701 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.774037] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39ca6da5-e22a-404d-a3f2-e432bfc9ea76 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.804344] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6660895-5fe0-4c43-a073-c71f8d136072 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.811516] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1db2e579-5d98-4612-b7fb-a4812671cfd4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.825104] env[64020]: DEBUG nova.compute.provider_tree [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 870.825626] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 1eddacecf24b470aab74de9e92e0e31b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 870.833128] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1eddacecf24b470aab74de9e92e0e31b [ 870.919212] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Releasing lock "refresh_cache-f4682b82-7475-412b-9319-bee1f05a9c63" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 870.919642] env[64020]: DEBUG nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 870.919868] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 870.920176] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-eccd10f1-c5ec-4576-a607-03318b3558ef {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.928799] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ca8c730-8baf-499f-9319-d80a43cb4f2c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.948922] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f4682b82-7475-412b-9319-bee1f05a9c63 could not be found. [ 870.949144] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 870.949400] env[64020]: INFO nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Took 0.03 seconds to destroy the instance on the hypervisor. [ 870.949642] env[64020]: DEBUG oslo.service.loopingcall [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 870.949839] env[64020]: DEBUG nova.compute.manager [-] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 870.949931] env[64020]: DEBUG nova.network.neutron [-] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 870.964347] env[64020]: DEBUG nova.network.neutron [-] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 870.964813] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0e8aa26782154daa83a4b2351fa75a31 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 870.971229] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e8aa26782154daa83a4b2351fa75a31 [ 871.088086] env[64020]: DEBUG nova.network.neutron [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 871.088633] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 7dfc978910554fb9bb1deacc6d37e0da in queue reply_57893177120949e6a93cb88e15cd42b4 [ 871.096477] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7dfc978910554fb9bb1deacc6d37e0da [ 871.245504] env[64020]: DEBUG nova.compute.manager [req-684de6c9-4734-4996-bfcd-08102405b1d9 req-f8559b44-20ea-4ac9-a3d1-af9549b415ca service nova] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Received event network-changed-842426d0-e222-405d-a943-e75a3af7e696 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 871.245703] env[64020]: DEBUG nova.compute.manager [req-684de6c9-4734-4996-bfcd-08102405b1d9 req-f8559b44-20ea-4ac9-a3d1-af9549b415ca service nova] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Refreshing instance network info cache due to event network-changed-842426d0-e222-405d-a943-e75a3af7e696. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 871.245912] env[64020]: DEBUG oslo_concurrency.lockutils [req-684de6c9-4734-4996-bfcd-08102405b1d9 req-f8559b44-20ea-4ac9-a3d1-af9549b415ca service nova] Acquiring lock "refresh_cache-f4682b82-7475-412b-9319-bee1f05a9c63" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 871.246052] env[64020]: DEBUG oslo_concurrency.lockutils [req-684de6c9-4734-4996-bfcd-08102405b1d9 req-f8559b44-20ea-4ac9-a3d1-af9549b415ca service nova] Acquired lock "refresh_cache-f4682b82-7475-412b-9319-bee1f05a9c63" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 871.246203] env[64020]: DEBUG nova.network.neutron [req-684de6c9-4734-4996-bfcd-08102405b1d9 req-f8559b44-20ea-4ac9-a3d1-af9549b415ca service nova] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Refreshing network info cache for port 842426d0-e222-405d-a943-e75a3af7e696 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 871.246639] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-684de6c9-4734-4996-bfcd-08102405b1d9 req-f8559b44-20ea-4ac9-a3d1-af9549b415ca service nova] Expecting reply to msg 852e9bd896b74a4eb28c82b4a4d5c354 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 871.257011] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 852e9bd896b74a4eb28c82b4a4d5c354 [ 871.328185] env[64020]: DEBUG nova.scheduler.client.report [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 871.330588] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 47fab789e8084cf8ad96c7436fadd653 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 871.344224] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47fab789e8084cf8ad96c7436fadd653 [ 871.466593] env[64020]: DEBUG nova.network.neutron [-] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 871.467060] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg e9a685207f3f4cc6a479ce74c33551d0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 871.475849] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9a685207f3f4cc6a479ce74c33551d0 [ 871.590871] env[64020]: INFO nova.compute.manager [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] [instance: 45167852-b7c0-4614-89f1-f8f7fc2078f2] Took 1.02 seconds to deallocate network for instance. [ 871.592887] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg dd6e9309180245c6ae95dcc7058d6f87 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 871.626857] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dd6e9309180245c6ae95dcc7058d6f87 [ 871.772748] env[64020]: DEBUG nova.network.neutron [req-684de6c9-4734-4996-bfcd-08102405b1d9 req-f8559b44-20ea-4ac9-a3d1-af9549b415ca service nova] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 871.817923] env[64020]: DEBUG nova.network.neutron [req-684de6c9-4734-4996-bfcd-08102405b1d9 req-f8559b44-20ea-4ac9-a3d1-af9549b415ca service nova] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 871.819118] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-684de6c9-4734-4996-bfcd-08102405b1d9 req-f8559b44-20ea-4ac9-a3d1-af9549b415ca service nova] Expecting reply to msg 19d1846501f34f0ea9c727bf6b65f309 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 871.826853] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19d1846501f34f0ea9c727bf6b65f309 [ 871.832714] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.368s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 871.833205] env[64020]: DEBUG nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 871.834931] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 4755dbd41e9a4f0c807fa67f06459dfb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 871.836553] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.555s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 871.838328] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 8fae2491cd394e0aa26f2b4f82e89e0c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 871.864870] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4755dbd41e9a4f0c807fa67f06459dfb [ 871.875162] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8fae2491cd394e0aa26f2b4f82e89e0c [ 871.969023] env[64020]: INFO nova.compute.manager [-] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Took 1.02 seconds to deallocate network for instance. [ 871.972376] env[64020]: DEBUG nova.compute.claims [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 871.972376] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 872.097923] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg e1af89d02bb54a108315b160d1166bb6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 872.127609] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e1af89d02bb54a108315b160d1166bb6 [ 872.321353] env[64020]: DEBUG oslo_concurrency.lockutils [req-684de6c9-4734-4996-bfcd-08102405b1d9 req-f8559b44-20ea-4ac9-a3d1-af9549b415ca service nova] Releasing lock "refresh_cache-f4682b82-7475-412b-9319-bee1f05a9c63" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 872.321353] env[64020]: DEBUG nova.compute.manager [req-684de6c9-4734-4996-bfcd-08102405b1d9 req-f8559b44-20ea-4ac9-a3d1-af9549b415ca service nova] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Received event network-vif-deleted-842426d0-e222-405d-a943-e75a3af7e696 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 872.341955] env[64020]: DEBUG nova.compute.utils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 872.342505] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 9de67eb1f70e4d22881a699fbe07c830 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 872.347416] env[64020]: DEBUG nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 872.347416] env[64020]: DEBUG nova.network.neutron [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 872.354007] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9de67eb1f70e4d22881a699fbe07c830 [ 872.383557] env[64020]: DEBUG nova.policy [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2e612db51c8a49b3b942ac70e24025f6', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd139e605a6f54506ab7b94cc56578734', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 872.619569] env[64020]: INFO nova.scheduler.client.report [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Deleted allocations for instance 45167852-b7c0-4614-89f1-f8f7fc2078f2 [ 872.629176] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Expecting reply to msg 696c1a52d88e4e3cb216382759092e01 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 872.630763] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b0c8353-0629-4ab2-b20f-f64f13ad7d37 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.638448] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e30e9a82-958e-4b81-9a6d-d91e876d592e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.669968] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 696c1a52d88e4e3cb216382759092e01 [ 872.671056] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31518bfe-cbb7-43b9-9875-8e5e193b6ddc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.678873] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55dd8d96-b968-4301-b2b8-e0f509d1b4b9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.693432] env[64020]: DEBUG nova.compute.provider_tree [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 872.693924] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg a88384fb833e46be917a5f0ea31fb374 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 872.702860] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a88384fb833e46be917a5f0ea31fb374 [ 872.755921] env[64020]: DEBUG nova.network.neutron [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Successfully created port: b67bb76f-4607-418d-8c89-c6cae26a7466 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 872.847913] env[64020]: DEBUG nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 872.850594] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg bf8346a815d149a9bea79f0565e10dab in queue reply_57893177120949e6a93cb88e15cd42b4 [ 872.883334] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf8346a815d149a9bea79f0565e10dab [ 873.136422] env[64020]: DEBUG oslo_concurrency.lockutils [None req-79a5a3d7-023d-4204-bd6b-4254f8950210 tempest-ServerPasswordTestJSON-557696207 tempest-ServerPasswordTestJSON-557696207-project-member] Lock "45167852-b7c0-4614-89f1-f8f7fc2078f2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 178.880s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 873.137027] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 38810c18093e453086b8c9fbfe0b0c35 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 873.146284] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 38810c18093e453086b8c9fbfe0b0c35 [ 873.196115] env[64020]: DEBUG nova.scheduler.client.report [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 873.198536] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 154375c6580a448eb5edcc99bd9256be in queue reply_57893177120949e6a93cb88e15cd42b4 [ 873.210342] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 154375c6580a448eb5edcc99bd9256be [ 873.355201] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 682b3cf21e3f4fa7a339d79793655cb2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 873.387259] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 682b3cf21e3f4fa7a339d79793655cb2 [ 873.638973] env[64020]: DEBUG nova.compute.manager [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 873.641084] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg b0454f34f83b4bfbbd63341581224f76 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 873.674572] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b0454f34f83b4bfbbd63341581224f76 [ 873.703422] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.867s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 873.704203] env[64020]: ERROR nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6a31c3b4-3e9f-44ef-a054-d0e46cbcd727, please check neutron logs for more information. [ 873.704203] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Traceback (most recent call last): [ 873.704203] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 873.704203] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] self.driver.spawn(context, instance, image_meta, [ 873.704203] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 873.704203] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 873.704203] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 873.704203] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] vm_ref = self.build_virtual_machine(instance, [ 873.704203] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 873.704203] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] vif_infos = vmwarevif.get_vif_info(self._session, [ 873.704203] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] for vif in network_info: [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] return self._sync_wrapper(fn, *args, **kwargs) [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] self.wait() [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] self[:] = self._gt.wait() [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] return self._exit_event.wait() [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] result = hub.switch() [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 873.704576] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] return self.greenlet.switch() [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] result = function(*args, **kwargs) [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] return func(*args, **kwargs) [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] raise e [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] nwinfo = self.network_api.allocate_for_instance( [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] created_port_ids = self._update_ports_for_instance( [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] with excutils.save_and_reraise_exception(): [ 873.704941] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 873.705316] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] self.force_reraise() [ 873.705316] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 873.705316] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] raise self.value [ 873.705316] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 873.705316] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] updated_port = self._update_port( [ 873.705316] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 873.705316] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] _ensure_no_port_binding_failure(port) [ 873.705316] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 873.705316] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] raise exception.PortBindingFailed(port_id=port['id']) [ 873.705316] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] nova.exception.PortBindingFailed: Binding failed for port 6a31c3b4-3e9f-44ef-a054-d0e46cbcd727, please check neutron logs for more information. [ 873.705316] env[64020]: ERROR nova.compute.manager [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] [ 873.705616] env[64020]: DEBUG nova.compute.utils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Binding failed for port 6a31c3b4-3e9f-44ef-a054-d0e46cbcd727, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 873.706128] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.195s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 873.708137] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 172c62152914414d852cd0d487957301 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 873.709184] env[64020]: DEBUG nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Build of instance d567d3dd-5820-4d6c-8922-3e39a1d9ebd6 was re-scheduled: Binding failed for port 6a31c3b4-3e9f-44ef-a054-d0e46cbcd727, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 873.709740] env[64020]: DEBUG nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 873.709968] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquiring lock "refresh_cache-d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 873.710113] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquired lock "refresh_cache-d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 873.710276] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 873.710643] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg daa566b7e2bf47b3a2040b339a6db787 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 873.725021] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg daa566b7e2bf47b3a2040b339a6db787 [ 873.742451] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 172c62152914414d852cd0d487957301 [ 873.745492] env[64020]: DEBUG nova.compute.manager [req-3fd52556-dd6c-44cb-9704-43f2919d3293 req-9b4e725b-db4b-4a1e-b48b-8472ac10f5a8 service nova] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Received event network-changed-b67bb76f-4607-418d-8c89-c6cae26a7466 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 873.745715] env[64020]: DEBUG nova.compute.manager [req-3fd52556-dd6c-44cb-9704-43f2919d3293 req-9b4e725b-db4b-4a1e-b48b-8472ac10f5a8 service nova] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Refreshing instance network info cache due to event network-changed-b67bb76f-4607-418d-8c89-c6cae26a7466. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 873.745932] env[64020]: DEBUG oslo_concurrency.lockutils [req-3fd52556-dd6c-44cb-9704-43f2919d3293 req-9b4e725b-db4b-4a1e-b48b-8472ac10f5a8 service nova] Acquiring lock "refresh_cache-1235c526-3cfb-42b0-8e2d-64cfc59ec35e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 873.746081] env[64020]: DEBUG oslo_concurrency.lockutils [req-3fd52556-dd6c-44cb-9704-43f2919d3293 req-9b4e725b-db4b-4a1e-b48b-8472ac10f5a8 service nova] Acquired lock "refresh_cache-1235c526-3cfb-42b0-8e2d-64cfc59ec35e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 873.746225] env[64020]: DEBUG nova.network.neutron [req-3fd52556-dd6c-44cb-9704-43f2919d3293 req-9b4e725b-db4b-4a1e-b48b-8472ac10f5a8 service nova] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Refreshing network info cache for port b67bb76f-4607-418d-8c89-c6cae26a7466 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 873.746642] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3fd52556-dd6c-44cb-9704-43f2919d3293 req-9b4e725b-db4b-4a1e-b48b-8472ac10f5a8 service nova] Expecting reply to msg c8ee14ee138745db81e7b8053294b66d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 873.752336] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c8ee14ee138745db81e7b8053294b66d [ 873.865628] env[64020]: DEBUG nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 873.906341] env[64020]: ERROR nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b67bb76f-4607-418d-8c89-c6cae26a7466, please check neutron logs for more information. [ 873.906341] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 873.906341] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 873.906341] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 873.906341] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 873.906341] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 873.906341] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 873.906341] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 873.906341] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 873.906341] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 873.906341] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 873.906341] env[64020]: ERROR nova.compute.manager raise self.value [ 873.906341] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 873.906341] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 873.906341] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 873.906341] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 873.906863] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 873.906863] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 873.906863] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b67bb76f-4607-418d-8c89-c6cae26a7466, please check neutron logs for more information. [ 873.906863] env[64020]: ERROR nova.compute.manager [ 873.907516] env[64020]: Traceback (most recent call last): [ 873.907657] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 873.907657] env[64020]: listener.cb(fileno) [ 873.907872] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 873.907872] env[64020]: result = function(*args, **kwargs) [ 873.908050] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 873.908050] env[64020]: return func(*args, **kwargs) [ 873.908160] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 873.908160] env[64020]: raise e [ 873.908255] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 873.908255] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 873.908347] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 873.908347] env[64020]: created_port_ids = self._update_ports_for_instance( [ 873.908433] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 873.908433] env[64020]: with excutils.save_and_reraise_exception(): [ 873.908514] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 873.908514] env[64020]: self.force_reraise() [ 873.908579] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 873.908579] env[64020]: raise self.value [ 873.908641] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 873.908641] env[64020]: updated_port = self._update_port( [ 873.908697] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 873.908697] env[64020]: _ensure_no_port_binding_failure(port) [ 873.908819] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 873.908819] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 873.908892] env[64020]: nova.exception.PortBindingFailed: Binding failed for port b67bb76f-4607-418d-8c89-c6cae26a7466, please check neutron logs for more information. [ 873.908942] env[64020]: Removing descriptor: 16 [ 873.916866] env[64020]: DEBUG nova.virt.hardware [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 873.917382] env[64020]: DEBUG nova.virt.hardware [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 873.917701] env[64020]: DEBUG nova.virt.hardware [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 873.918163] env[64020]: DEBUG nova.virt.hardware [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 873.918473] env[64020]: DEBUG nova.virt.hardware [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 873.918946] env[64020]: DEBUG nova.virt.hardware [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 873.919323] env[64020]: DEBUG nova.virt.hardware [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 873.919642] env[64020]: DEBUG nova.virt.hardware [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 873.919980] env[64020]: DEBUG nova.virt.hardware [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 873.920468] env[64020]: DEBUG nova.virt.hardware [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 873.920821] env[64020]: DEBUG nova.virt.hardware [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 873.922107] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5c710a9e-91d2-4c31-9e32-e1a662f8c678 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 873.932459] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b26a42b7-946e-4ce0-b5e4-d77830cc1bac {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 873.948794] env[64020]: ERROR nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b67bb76f-4607-418d-8c89-c6cae26a7466, please check neutron logs for more information. [ 873.948794] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Traceback (most recent call last): [ 873.948794] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 873.948794] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] yield resources [ 873.948794] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 873.948794] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] self.driver.spawn(context, instance, image_meta, [ 873.948794] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 873.948794] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 873.948794] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 873.948794] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] vm_ref = self.build_virtual_machine(instance, [ 873.948794] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] vif_infos = vmwarevif.get_vif_info(self._session, [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] for vif in network_info: [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] return self._sync_wrapper(fn, *args, **kwargs) [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] self.wait() [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] self[:] = self._gt.wait() [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] return self._exit_event.wait() [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 873.949193] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] current.throw(*self._exc) [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] result = function(*args, **kwargs) [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] return func(*args, **kwargs) [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] raise e [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] nwinfo = self.network_api.allocate_for_instance( [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] created_port_ids = self._update_ports_for_instance( [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] with excutils.save_and_reraise_exception(): [ 873.949547] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 873.949921] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] self.force_reraise() [ 873.949921] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 873.949921] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] raise self.value [ 873.949921] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 873.949921] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] updated_port = self._update_port( [ 873.949921] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 873.949921] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] _ensure_no_port_binding_failure(port) [ 873.949921] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 873.949921] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] raise exception.PortBindingFailed(port_id=port['id']) [ 873.949921] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] nova.exception.PortBindingFailed: Binding failed for port b67bb76f-4607-418d-8c89-c6cae26a7466, please check neutron logs for more information. [ 873.949921] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] [ 873.950664] env[64020]: INFO nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Terminating instance [ 873.952912] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "refresh_cache-1235c526-3cfb-42b0-8e2d-64cfc59ec35e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 874.161120] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 874.234036] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 874.266167] env[64020]: DEBUG nova.network.neutron [req-3fd52556-dd6c-44cb-9704-43f2919d3293 req-9b4e725b-db4b-4a1e-b48b-8472ac10f5a8 service nova] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 874.314297] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 874.315033] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 2cd7a27ea3d940029af452f787c841e0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 874.323462] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2cd7a27ea3d940029af452f787c841e0 [ 874.411335] env[64020]: DEBUG nova.network.neutron [req-3fd52556-dd6c-44cb-9704-43f2919d3293 req-9b4e725b-db4b-4a1e-b48b-8472ac10f5a8 service nova] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 874.411939] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3fd52556-dd6c-44cb-9704-43f2919d3293 req-9b4e725b-db4b-4a1e-b48b-8472ac10f5a8 service nova] Expecting reply to msg c660e932d2194a019bc87e0f22141f26 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 874.419960] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c660e932d2194a019bc87e0f22141f26 [ 874.535374] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d75172d9-e1c1-4c9e-b3f3-31290fe9653f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.554103] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-130f886e-470a-4158-9da5-aa23d08a8702 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.589764] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f2c93f9-e6ca-4be3-b1de-61742224e7d4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.597388] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ecba568d-f55b-4f1e-a0e2-c3152a1606fd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.610525] env[64020]: DEBUG nova.compute.provider_tree [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 874.611576] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 177c66ed8c884897aa7609ec8cc620bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 874.619163] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 177c66ed8c884897aa7609ec8cc620bc [ 874.817377] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Releasing lock "refresh_cache-d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 874.817674] env[64020]: DEBUG nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 874.817784] env[64020]: DEBUG nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 874.817949] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 874.836169] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 874.836718] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg b71fb03eba574cf8b288d01a0cf49706 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 874.843514] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b71fb03eba574cf8b288d01a0cf49706 [ 874.917547] env[64020]: DEBUG oslo_concurrency.lockutils [req-3fd52556-dd6c-44cb-9704-43f2919d3293 req-9b4e725b-db4b-4a1e-b48b-8472ac10f5a8 service nova] Releasing lock "refresh_cache-1235c526-3cfb-42b0-8e2d-64cfc59ec35e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 874.918149] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquired lock "refresh_cache-1235c526-3cfb-42b0-8e2d-64cfc59ec35e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 874.918450] env[64020]: DEBUG nova.network.neutron [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 874.919181] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 14ce605963cf423fb98288b700c0cc6a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 874.925809] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14ce605963cf423fb98288b700c0cc6a [ 875.114631] env[64020]: DEBUG nova.scheduler.client.report [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 875.117253] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 3a93bf86db2e421385e95486a701ee5b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 875.133890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3a93bf86db2e421385e95486a701ee5b [ 875.338984] env[64020]: DEBUG nova.network.neutron [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 875.339528] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 1e2ff921a5344608b8e48ce522aacf26 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 875.348734] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e2ff921a5344608b8e48ce522aacf26 [ 875.437999] env[64020]: DEBUG nova.network.neutron [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 875.528732] env[64020]: DEBUG nova.network.neutron [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 875.529355] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg be255ab36612479d882b3f0ea0e845de in queue reply_57893177120949e6a93cb88e15cd42b4 [ 875.537140] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be255ab36612479d882b3f0ea0e845de [ 875.620333] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.914s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 875.621062] env[64020]: ERROR nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 53782645-b8fb-4e78-bdaf-8f397474785d, please check neutron logs for more information. [ 875.621062] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] Traceback (most recent call last): [ 875.621062] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 875.621062] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] self.driver.spawn(context, instance, image_meta, [ 875.621062] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 875.621062] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] self._vmops.spawn(context, instance, image_meta, injected_files, [ 875.621062] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 875.621062] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] vm_ref = self.build_virtual_machine(instance, [ 875.621062] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 875.621062] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] vif_infos = vmwarevif.get_vif_info(self._session, [ 875.621062] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] for vif in network_info: [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] return self._sync_wrapper(fn, *args, **kwargs) [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] self.wait() [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] self[:] = self._gt.wait() [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] return self._exit_event.wait() [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] result = hub.switch() [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 875.621393] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] return self.greenlet.switch() [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] result = function(*args, **kwargs) [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] return func(*args, **kwargs) [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] raise e [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] nwinfo = self.network_api.allocate_for_instance( [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] created_port_ids = self._update_ports_for_instance( [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] with excutils.save_and_reraise_exception(): [ 875.621706] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 875.622023] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] self.force_reraise() [ 875.622023] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 875.622023] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] raise self.value [ 875.622023] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 875.622023] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] updated_port = self._update_port( [ 875.622023] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 875.622023] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] _ensure_no_port_binding_failure(port) [ 875.622023] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 875.622023] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] raise exception.PortBindingFailed(port_id=port['id']) [ 875.622023] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] nova.exception.PortBindingFailed: Binding failed for port 53782645-b8fb-4e78-bdaf-8f397474785d, please check neutron logs for more information. [ 875.622023] env[64020]: ERROR nova.compute.manager [instance: ecb7f281-1206-4693-845b-c59f639b0789] [ 875.622657] env[64020]: DEBUG nova.compute.utils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Binding failed for port 53782645-b8fb-4e78-bdaf-8f397474785d, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 875.624366] env[64020]: DEBUG nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Build of instance ecb7f281-1206-4693-845b-c59f639b0789 was re-scheduled: Binding failed for port 53782645-b8fb-4e78-bdaf-8f397474785d, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 875.624840] env[64020]: DEBUG nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 875.625121] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-ecb7f281-1206-4693-845b-c59f639b0789" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 875.625326] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-ecb7f281-1206-4693-845b-c59f639b0789" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 875.625767] env[64020]: DEBUG nova.network.neutron [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 875.627290] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 4e3fecb6bbf4438d9ac6c42bae271bc9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 875.628506] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.438s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 875.630793] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 32841f30208447f2a114d7688a2b0d17 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 875.634101] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e3fecb6bbf4438d9ac6c42bae271bc9 [ 875.669843] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 32841f30208447f2a114d7688a2b0d17 [ 875.771204] env[64020]: DEBUG nova.compute.manager [req-2c0aeb27-84cf-455d-b93b-b58e06578318 req-6f0d5244-07ee-46e0-835d-fc3170af506b service nova] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Received event network-vif-deleted-b67bb76f-4607-418d-8c89-c6cae26a7466 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 875.844688] env[64020]: INFO nova.compute.manager [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: d567d3dd-5820-4d6c-8922-3e39a1d9ebd6] Took 1.03 seconds to deallocate network for instance. [ 875.846408] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 450204587d4245c8826e5fa00a875059 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 875.881528] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 450204587d4245c8826e5fa00a875059 [ 876.031552] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Releasing lock "refresh_cache-1235c526-3cfb-42b0-8e2d-64cfc59ec35e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 876.031982] env[64020]: DEBUG nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 876.032205] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 876.032513] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3b186fc8-57f1-4d25-9d92-7f493005314f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 876.041956] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0cb8d393-3a35-493f-aa0a-18eefe758491 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 876.063938] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1235c526-3cfb-42b0-8e2d-64cfc59ec35e could not be found. [ 876.064168] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 876.064344] env[64020]: INFO nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Took 0.03 seconds to destroy the instance on the hypervisor. [ 876.064581] env[64020]: DEBUG oslo.service.loopingcall [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 876.064794] env[64020]: DEBUG nova.compute.manager [-] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 876.064879] env[64020]: DEBUG nova.network.neutron [-] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 876.080801] env[64020]: DEBUG nova.network.neutron [-] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 876.080801] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 087dfc7774e644098a7c95a71465e5b5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 876.087193] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 087dfc7774e644098a7c95a71465e5b5 [ 876.149524] env[64020]: DEBUG nova.network.neutron [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 876.263236] env[64020]: DEBUG nova.network.neutron [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 876.271468] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 3daf0e95422d4c8ca2067a7a14ad4297 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 876.279490] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3daf0e95422d4c8ca2067a7a14ad4297 [ 876.350878] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 8ecf2b2c9a2643cda79eb457a74f9b09 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 876.383008] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ecf2b2c9a2643cda79eb457a74f9b09 [ 876.466020] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f76a74ee-929c-40b2-a192-1afe23807849 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 876.473136] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afae762e-07ce-4299-9791-27938ee2f332 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 876.506399] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13cfe063-8718-4241-94fb-366464eb12f2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 876.513752] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af7ff133-5ccc-4c95-ac0e-0929173b0197 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 876.527311] env[64020]: DEBUG nova.compute.provider_tree [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 876.527892] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg f29b5decce2641069e1f613d720948ac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 876.534953] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f29b5decce2641069e1f613d720948ac [ 876.584031] env[64020]: DEBUG nova.network.neutron [-] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 876.584031] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b3dcb60834b943a5a6db255d8e5b531d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 876.592270] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3dcb60834b943a5a6db255d8e5b531d [ 876.773673] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-ecb7f281-1206-4693-845b-c59f639b0789" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 876.773924] env[64020]: DEBUG nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 876.774106] env[64020]: DEBUG nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 876.774269] env[64020]: DEBUG nova.network.neutron [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 876.788632] env[64020]: DEBUG nova.network.neutron [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 876.789147] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg d1181dd61e5c4f159d371f630ba42130 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 876.795992] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1181dd61e5c4f159d371f630ba42130 [ 876.872030] env[64020]: INFO nova.scheduler.client.report [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Deleted allocations for instance d567d3dd-5820-4d6c-8922-3e39a1d9ebd6 [ 876.879074] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 18ec8e5358a34fa882fdaba4b31cdf20 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 876.895428] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18ec8e5358a34fa882fdaba4b31cdf20 [ 877.031683] env[64020]: DEBUG nova.scheduler.client.report [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 877.034233] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg aa894a29e48640cdb6fec34e49496704 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 877.048290] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa894a29e48640cdb6fec34e49496704 [ 877.086626] env[64020]: INFO nova.compute.manager [-] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Took 1.02 seconds to deallocate network for instance. [ 877.088172] env[64020]: DEBUG nova.compute.claims [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 877.088353] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 877.291304] env[64020]: DEBUG nova.network.neutron [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 877.291868] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg fce3317d1e334945ab2375c06f6c719b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 877.301092] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fce3317d1e334945ab2375c06f6c719b [ 877.381567] env[64020]: DEBUG oslo_concurrency.lockutils [None req-875d499e-6b5e-4159-b3ef-290d58bd5b9b tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "d567d3dd-5820-4d6c-8922-3e39a1d9ebd6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 167.737s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 877.382043] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 0eb53c271f6a417aae14f427291ddf50 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 877.390871] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0eb53c271f6a417aae14f427291ddf50 [ 877.537280] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.908s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 877.538272] env[64020]: ERROR nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 7781f67c-a182-4566-a351-61cb71e27082, please check neutron logs for more information. [ 877.538272] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Traceback (most recent call last): [ 877.538272] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 877.538272] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] self.driver.spawn(context, instance, image_meta, [ 877.538272] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 877.538272] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 877.538272] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 877.538272] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] vm_ref = self.build_virtual_machine(instance, [ 877.538272] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 877.538272] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] vif_infos = vmwarevif.get_vif_info(self._session, [ 877.538272] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] for vif in network_info: [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] return self._sync_wrapper(fn, *args, **kwargs) [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] self.wait() [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] self[:] = self._gt.wait() [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] return self._exit_event.wait() [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] current.throw(*self._exc) [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 877.538614] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] result = function(*args, **kwargs) [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] return func(*args, **kwargs) [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] raise e [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] nwinfo = self.network_api.allocate_for_instance( [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] created_port_ids = self._update_ports_for_instance( [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] with excutils.save_and_reraise_exception(): [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] self.force_reraise() [ 877.538982] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 877.539336] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] raise self.value [ 877.539336] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 877.539336] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] updated_port = self._update_port( [ 877.539336] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 877.539336] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] _ensure_no_port_binding_failure(port) [ 877.539336] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 877.539336] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] raise exception.PortBindingFailed(port_id=port['id']) [ 877.539336] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] nova.exception.PortBindingFailed: Binding failed for port 7781f67c-a182-4566-a351-61cb71e27082, please check neutron logs for more information. [ 877.539336] env[64020]: ERROR nova.compute.manager [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] [ 877.539336] env[64020]: DEBUG nova.compute.utils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Binding failed for port 7781f67c-a182-4566-a351-61cb71e27082, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 877.541195] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.735s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 877.543249] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 5ad100f0a5dc43bc8953ed178f10dec8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 877.545172] env[64020]: DEBUG nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Build of instance 8de44ad9-adb7-4589-b2d8-a2aeeb89892f was re-scheduled: Binding failed for port 7781f67c-a182-4566-a351-61cb71e27082, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 877.545604] env[64020]: DEBUG nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 877.545828] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquiring lock "refresh_cache-8de44ad9-adb7-4589-b2d8-a2aeeb89892f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 877.545973] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Acquired lock "refresh_cache-8de44ad9-adb7-4589-b2d8-a2aeeb89892f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 877.546128] env[64020]: DEBUG nova.network.neutron [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 877.546502] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg fee48ef9066442ca80ab6d6b877af965 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 877.553592] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fee48ef9066442ca80ab6d6b877af965 [ 877.574855] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ad100f0a5dc43bc8953ed178f10dec8 [ 877.794068] env[64020]: INFO nova.compute.manager [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: ecb7f281-1206-4693-845b-c59f639b0789] Took 1.02 seconds to deallocate network for instance. [ 877.795751] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg d46704cc010b4e87bda30a3e6c75d4df in queue reply_57893177120949e6a93cb88e15cd42b4 [ 877.826222] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d46704cc010b4e87bda30a3e6c75d4df [ 877.883943] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 877.886081] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 9083e9125bd54db4b8ec3bf5a58089ea in queue reply_57893177120949e6a93cb88e15cd42b4 [ 877.917409] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9083e9125bd54db4b8ec3bf5a58089ea [ 878.069428] env[64020]: DEBUG nova.network.neutron [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 878.157436] env[64020]: DEBUG nova.network.neutron [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 878.157975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 4554d33a440f4223b70f957b37024986 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 878.166706] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4554d33a440f4223b70f957b37024986 [ 878.300730] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 5602f84bf48f40a2837a792110d98941 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 878.322147] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa9dcb31-52ad-4297-af95-68d232f29a35 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.329723] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16ff2f5d-7aad-4039-bedf-23da311f4f11 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.333322] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5602f84bf48f40a2837a792110d98941 [ 878.360846] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f416b7b-85f5-4994-aab1-f5c7055c0f24 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.369565] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80795700-3a13-4161-b163-4c23b51b6bf7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.391996] env[64020]: DEBUG nova.compute.provider_tree [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 878.392535] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 59136291606243a6b219f373ddeb43bf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 878.405793] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59136291606243a6b219f373ddeb43bf [ 878.407299] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 878.662584] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Releasing lock "refresh_cache-8de44ad9-adb7-4589-b2d8-a2aeeb89892f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 878.662828] env[64020]: DEBUG nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 878.663010] env[64020]: DEBUG nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 878.663178] env[64020]: DEBUG nova.network.neutron [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 878.693045] env[64020]: DEBUG nova.network.neutron [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 878.693045] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 4fa9866241a2491da2ccf0769ffbd58e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 878.707496] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4fa9866241a2491da2ccf0769ffbd58e [ 878.822458] env[64020]: INFO nova.scheduler.client.report [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Deleted allocations for instance ecb7f281-1206-4693-845b-c59f639b0789 [ 878.828841] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 3ebd6058b9e144fb96e268ce0d683b8f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 878.842858] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3ebd6058b9e144fb96e268ce0d683b8f [ 878.896757] env[64020]: DEBUG nova.scheduler.client.report [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 878.899150] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg c316454b713c4229a48c53df74ae6f3d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 878.911975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c316454b713c4229a48c53df74ae6f3d [ 879.202397] env[64020]: DEBUG nova.network.neutron [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 879.202938] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 5ddf821619134a359e64905dd57c955e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 879.211036] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ddf821619134a359e64905dd57c955e [ 879.330914] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fd0ac052-50d8-4793-bf30-50c65fd67b88 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "ecb7f281-1206-4693-845b-c59f639b0789" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 164.094s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 879.331534] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg b5e35caa02dd48369fc58aadc9d6eddf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 879.342922] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b5e35caa02dd48369fc58aadc9d6eddf [ 879.401814] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.861s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 879.402487] env[64020]: ERROR nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d6820d39-d605-4caf-a814-ff1498a867f6, please check neutron logs for more information. [ 879.402487] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Traceback (most recent call last): [ 879.402487] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 879.402487] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] self.driver.spawn(context, instance, image_meta, [ 879.402487] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 879.402487] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] self._vmops.spawn(context, instance, image_meta, injected_files, [ 879.402487] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 879.402487] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] vm_ref = self.build_virtual_machine(instance, [ 879.402487] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 879.402487] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] vif_infos = vmwarevif.get_vif_info(self._session, [ 879.402487] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] for vif in network_info: [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] return self._sync_wrapper(fn, *args, **kwargs) [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] self.wait() [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] self[:] = self._gt.wait() [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] return self._exit_event.wait() [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] current.throw(*self._exc) [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 879.402763] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] result = function(*args, **kwargs) [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] return func(*args, **kwargs) [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] raise e [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] nwinfo = self.network_api.allocate_for_instance( [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] created_port_ids = self._update_ports_for_instance( [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] with excutils.save_and_reraise_exception(): [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] self.force_reraise() [ 879.403108] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 879.403433] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] raise self.value [ 879.403433] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 879.403433] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] updated_port = self._update_port( [ 879.403433] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 879.403433] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] _ensure_no_port_binding_failure(port) [ 879.403433] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 879.403433] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] raise exception.PortBindingFailed(port_id=port['id']) [ 879.403433] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] nova.exception.PortBindingFailed: Binding failed for port d6820d39-d605-4caf-a814-ff1498a867f6, please check neutron logs for more information. [ 879.403433] env[64020]: ERROR nova.compute.manager [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] [ 879.403433] env[64020]: DEBUG nova.compute.utils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Binding failed for port d6820d39-d605-4caf-a814-ff1498a867f6, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 879.404397] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.585s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 879.406935] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 5fdfafb1802441b3b97a1ec5a6076f33 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 879.407690] env[64020]: DEBUG nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Build of instance f7f8d239-aff8-4cd7-afdd-974c53b3e563 was re-scheduled: Binding failed for port d6820d39-d605-4caf-a814-ff1498a867f6, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 879.408101] env[64020]: DEBUG nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 879.408322] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquiring lock "refresh_cache-f7f8d239-aff8-4cd7-afdd-974c53b3e563" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 879.408467] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquired lock "refresh_cache-f7f8d239-aff8-4cd7-afdd-974c53b3e563" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 879.410873] env[64020]: DEBUG nova.network.neutron [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 879.410873] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 9c0c699718804472830d647e11e442dc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 879.416670] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c0c699718804472830d647e11e442dc [ 879.439800] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5fdfafb1802441b3b97a1ec5a6076f33 [ 879.705430] env[64020]: INFO nova.compute.manager [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] [instance: 8de44ad9-adb7-4589-b2d8-a2aeeb89892f] Took 1.04 seconds to deallocate network for instance. [ 879.707101] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 48716d11bb774e54b82b502e3c1bdb6d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 879.747507] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 48716d11bb774e54b82b502e3c1bdb6d [ 879.833437] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 879.835530] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 1ac8627a6175453b954307d6548d44c4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 879.872647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ac8627a6175453b954307d6548d44c4 [ 879.931844] env[64020]: DEBUG nova.network.neutron [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 880.035717] env[64020]: DEBUG nova.network.neutron [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 880.036293] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 4111ed6a8f1948848648153557e6be3e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 880.046120] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4111ed6a8f1948848648153557e6be3e [ 880.212722] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg 08727f5236254ff181bd185906bf9373 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 880.247023] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08727f5236254ff181bd185906bf9373 [ 880.297852] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04f04079-3099-4808-9e17-f3eb670549fb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 880.306210] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4a6f292-e1b3-472c-8347-397571f1ccb0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 880.336484] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e5e0cac-373a-417a-80ff-fa250114ca28 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 880.355220] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.356695] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb11db3a-e1be-4b51-a0ae-1b9e40483750 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 880.373519] env[64020]: DEBUG nova.compute.provider_tree [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 880.373519] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg d0ed86bda5b84df9bfa2362084e93e8d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 880.385229] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0ed86bda5b84df9bfa2362084e93e8d [ 880.487730] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "a2cd396e-bd03-4c51-8ec6-cd24d7541103" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.487730] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "a2cd396e-bd03-4c51-8ec6-cd24d7541103" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.508839] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquiring lock "5f485177-eb28-417d-a74c-0e0f30ea7ce2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.509225] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "5f485177-eb28-417d-a74c-0e0f30ea7ce2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.539298] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Releasing lock "refresh_cache-f7f8d239-aff8-4cd7-afdd-974c53b3e563" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 880.539696] env[64020]: DEBUG nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 880.540020] env[64020]: DEBUG nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 880.540330] env[64020]: DEBUG nova.network.neutron [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 880.555863] env[64020]: DEBUG nova.network.neutron [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 880.556765] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 0648458a7a554e66a32ab602a69d630f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 880.563712] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0648458a7a554e66a32ab602a69d630f [ 880.749032] env[64020]: INFO nova.scheduler.client.report [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Deleted allocations for instance 8de44ad9-adb7-4589-b2d8-a2aeeb89892f [ 880.752365] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Expecting reply to msg cc515cc213c540b8bbdb7c65a0726e3e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 880.766373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cc515cc213c540b8bbdb7c65a0726e3e [ 880.877728] env[64020]: DEBUG nova.scheduler.client.report [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 880.877728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 0b510f45502c4e2e9c800ffc0356fb98 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 880.890679] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b510f45502c4e2e9c800ffc0356fb98 [ 881.059189] env[64020]: DEBUG nova.network.neutron [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 881.059704] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 16bf273632e8422eb8eb5773f0a756f2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 881.071444] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16bf273632e8422eb8eb5773f0a756f2 [ 881.254991] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1dce71d0-4ff6-41de-a2e1-bdb4c19d9dc9 tempest-VolumesAdminNegativeTest-928207255 tempest-VolumesAdminNegativeTest-928207255-project-member] Lock "8de44ad9-adb7-4589-b2d8-a2aeeb89892f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 163.695s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 881.255633] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg d655a457f94043e29e1be4dc2e510ec5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 881.266454] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d655a457f94043e29e1be4dc2e510ec5 [ 881.380094] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.976s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 881.380767] env[64020]: ERROR nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 12c430b0-3ca9-4fe0-ad3d-30068930d0e7, please check neutron logs for more information. [ 881.380767] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Traceback (most recent call last): [ 881.380767] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 881.380767] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] self.driver.spawn(context, instance, image_meta, [ 881.380767] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 881.380767] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] self._vmops.spawn(context, instance, image_meta, injected_files, [ 881.380767] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 881.380767] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] vm_ref = self.build_virtual_machine(instance, [ 881.380767] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 881.380767] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] vif_infos = vmwarevif.get_vif_info(self._session, [ 881.380767] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] for vif in network_info: [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] return self._sync_wrapper(fn, *args, **kwargs) [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] self.wait() [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] self[:] = self._gt.wait() [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] return self._exit_event.wait() [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] result = hub.switch() [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 881.381312] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] return self.greenlet.switch() [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] result = function(*args, **kwargs) [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] return func(*args, **kwargs) [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] raise e [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] nwinfo = self.network_api.allocate_for_instance( [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] created_port_ids = self._update_ports_for_instance( [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] with excutils.save_and_reraise_exception(): [ 881.381976] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 881.382523] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] self.force_reraise() [ 881.382523] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 881.382523] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] raise self.value [ 881.382523] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 881.382523] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] updated_port = self._update_port( [ 881.382523] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 881.382523] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] _ensure_no_port_binding_failure(port) [ 881.382523] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 881.382523] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] raise exception.PortBindingFailed(port_id=port['id']) [ 881.382523] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] nova.exception.PortBindingFailed: Binding failed for port 12c430b0-3ca9-4fe0-ad3d-30068930d0e7, please check neutron logs for more information. [ 881.382523] env[64020]: ERROR nova.compute.manager [instance: dcb3a0b0-a558-45be-8208-038e465dc802] [ 881.382988] env[64020]: DEBUG nova.compute.utils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Binding failed for port 12c430b0-3ca9-4fe0-ad3d-30068930d0e7, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 881.382988] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.822s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 881.384927] env[64020]: INFO nova.compute.claims [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 881.386510] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 8f4a8c7e9d594d30853d2cb1979f813a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 881.398315] env[64020]: DEBUG nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Build of instance dcb3a0b0-a558-45be-8208-038e465dc802 was re-scheduled: Binding failed for port 12c430b0-3ca9-4fe0-ad3d-30068930d0e7, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 881.398786] env[64020]: DEBUG nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 881.399021] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "refresh_cache-dcb3a0b0-a558-45be-8208-038e465dc802" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 881.399157] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquired lock "refresh_cache-dcb3a0b0-a558-45be-8208-038e465dc802" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 881.399311] env[64020]: DEBUG nova.network.neutron [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 881.399741] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg c2b36eb1a9cb46b7bf0af8ab4ea998fa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 881.409248] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2b36eb1a9cb46b7bf0af8ab4ea998fa [ 881.430672] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f4a8c7e9d594d30853d2cb1979f813a [ 881.562428] env[64020]: INFO nova.compute.manager [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: f7f8d239-aff8-4cd7-afdd-974c53b3e563] Took 1.02 seconds to deallocate network for instance. [ 881.564272] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 95baebc2fad14b0eace935ca198666b4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 881.641811] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95baebc2fad14b0eace935ca198666b4 [ 881.758105] env[64020]: DEBUG nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 881.760060] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 4be249ce125a451e857e302b50e5899e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 881.790916] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4be249ce125a451e857e302b50e5899e [ 881.903264] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 52b3f9f5d26b4a7dacbbb2f7ce3b9055 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 881.914435] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52b3f9f5d26b4a7dacbbb2f7ce3b9055 [ 881.936705] env[64020]: DEBUG nova.network.neutron [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 882.038191] env[64020]: DEBUG nova.network.neutron [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 882.039543] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 7850f146f73a4478b4ca490b839a8084 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 882.052217] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7850f146f73a4478b4ca490b839a8084 [ 882.069564] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 9ee8aeadb46147ef9150a2439d872417 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 882.106195] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ee8aeadb46147ef9150a2439d872417 [ 882.289423] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.308689] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c094b819-adab-47e7-bd13-26d1e878d95d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.316957] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77b221ae-ec8c-4876-a2c4-578ab72457b1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.360675] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6715d7a-fa46-4d29-a0e3-edbc8a952632 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.367979] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ce0a713-35e7-4e40-a44f-c33af8586266 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.381424] env[64020]: DEBUG nova.compute.provider_tree [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 882.381915] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg afceb70c4c1548cea1c55ab71a4809a1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 882.388829] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg afceb70c4c1548cea1c55ab71a4809a1 [ 882.542266] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Releasing lock "refresh_cache-dcb3a0b0-a558-45be-8208-038e465dc802" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 882.542552] env[64020]: DEBUG nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 882.542738] env[64020]: DEBUG nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 882.542906] env[64020]: DEBUG nova.network.neutron [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 882.560980] env[64020]: DEBUG nova.network.neutron [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 882.561592] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg d9b6a31770d54895869863fad8e85b50 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 882.569935] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9b6a31770d54895869863fad8e85b50 [ 882.609323] env[64020]: INFO nova.scheduler.client.report [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Deleted allocations for instance f7f8d239-aff8-4cd7-afdd-974c53b3e563 [ 882.620057] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg c35eab43e178435ea2a46ad10ca82a63 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 882.635303] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c35eab43e178435ea2a46ad10ca82a63 [ 882.889214] env[64020]: DEBUG nova.scheduler.client.report [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 882.889214] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg b51d4629cafc48c185bab1357c3cbb58 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 882.900705] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b51d4629cafc48c185bab1357c3cbb58 [ 883.064954] env[64020]: DEBUG nova.network.neutron [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 883.064954] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 89cc306bd1f8436787390bedef79f3a4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 883.072651] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89cc306bd1f8436787390bedef79f3a4 [ 883.119091] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e82cf287-7f72-4ba4-a742-d3cb5d08f264 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "f7f8d239-aff8-4cd7-afdd-974c53b3e563" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 157.291s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 883.119091] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 288461a7cd6a43969286ad748c5cf2c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 883.129783] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 288461a7cd6a43969286ad748c5cf2c1 [ 883.396179] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.010s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 883.396179] env[64020]: DEBUG nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 883.396179] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 351a89add07e453a8b4d23f5bdd29555 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 883.397157] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 16.094s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 883.398107] env[64020]: DEBUG nova.objects.instance [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Lazy-loading 'resources' on Instance uuid 995288b5-d79a-4af3-a1e8-3571fff2d356 {{(pid=64020) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 883.398107] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 45c3b683172a4cf9b71fd18d8304adf1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 883.405750] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 45c3b683172a4cf9b71fd18d8304adf1 [ 883.430205] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 351a89add07e453a8b4d23f5bdd29555 [ 883.566158] env[64020]: INFO nova.compute.manager [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: dcb3a0b0-a558-45be-8208-038e465dc802] Took 1.02 seconds to deallocate network for instance. [ 883.567859] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg e74067b0a3c1474a88c6f72e42a4b5e2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 883.605055] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e74067b0a3c1474a88c6f72e42a4b5e2 [ 883.620161] env[64020]: DEBUG nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 883.621886] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 9f23001cae6940ddabcd40e49ccdb735 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 883.660118] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f23001cae6940ddabcd40e49ccdb735 [ 883.899234] env[64020]: DEBUG nova.compute.utils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 883.899885] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 5239b099b9ea4d229ec1ec1bdaa78009 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 883.903206] env[64020]: DEBUG nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 883.903381] env[64020]: DEBUG nova.network.neutron [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 883.919418] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5239b099b9ea4d229ec1ec1bdaa78009 [ 883.996017] env[64020]: DEBUG nova.policy [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8ce09bf5f89d47988c0fb8f3283ccc16', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0f4d60d9b3dd44c4a8c8ef58515191df', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 884.072385] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg e50efbe71c9a4477bcae0a9165c17b95 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 884.130325] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e50efbe71c9a4477bcae0a9165c17b95 [ 884.146922] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 884.264655] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-184ef0e4-0c19-4119-beb8-ffdd3734fb32 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.272649] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-898d5635-f23b-4b57-af7f-a397614681b7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.303854] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bca246ea-60ed-468d-ba23-09968b7f9efa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.311235] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-809c890a-d931-439a-83a2-6c9a1b02a02d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.324501] env[64020]: DEBUG nova.compute.provider_tree [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 884.324922] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 37fa9ab2c8524418a38529acba079b20 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 884.333391] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37fa9ab2c8524418a38529acba079b20 [ 884.407262] env[64020]: DEBUG nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 884.409002] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg ae50cdaeae9945d58b356ca520dcaf50 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 884.443637] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae50cdaeae9945d58b356ca520dcaf50 [ 884.503288] env[64020]: DEBUG nova.network.neutron [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Successfully created port: 3c981ae7-e113-416e-a449-8d35634800be {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 884.602245] env[64020]: INFO nova.scheduler.client.report [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Deleted allocations for instance dcb3a0b0-a558-45be-8208-038e465dc802 [ 884.608362] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 500277d8148147f58f444bf585ee24cf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 884.626729] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 500277d8148147f58f444bf585ee24cf [ 884.828116] env[64020]: DEBUG nova.scheduler.client.report [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 884.831230] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 57cb1286d0ce4eb2b2eeee7a99013f43 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 884.842512] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 57cb1286d0ce4eb2b2eeee7a99013f43 [ 884.914422] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 1ac7f5cd071a4485ad1b473c0d31f420 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 884.958095] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ac7f5cd071a4485ad1b473c0d31f420 [ 885.115546] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c64193d6-b6ca-4f8e-aeef-2f7641a13f74 tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "dcb3a0b0-a558-45be-8208-038e465dc802" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 139.798s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 885.115546] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg d7523d7ce0df4371a20883967848a515 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 885.127102] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7523d7ce0df4371a20883967848a515 [ 885.334051] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.937s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 885.337114] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.097s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 885.343647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg cf4927e581cb412eab4f2c1e4d7dfaed in queue reply_57893177120949e6a93cb88e15cd42b4 [ 885.367710] env[64020]: INFO nova.scheduler.client.report [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Deleted allocations for instance 995288b5-d79a-4af3-a1e8-3571fff2d356 [ 885.372332] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg 72bcd3308beb4793b3546f2ac7c40c49 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 885.376131] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cf4927e581cb412eab4f2c1e4d7dfaed [ 885.418444] env[64020]: DEBUG nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 885.422042] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72bcd3308beb4793b3546f2ac7c40c49 [ 885.460440] env[64020]: DEBUG nova.virt.hardware [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 885.460703] env[64020]: DEBUG nova.virt.hardware [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 885.460852] env[64020]: DEBUG nova.virt.hardware [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 885.461113] env[64020]: DEBUG nova.virt.hardware [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 885.461200] env[64020]: DEBUG nova.virt.hardware [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 885.461335] env[64020]: DEBUG nova.virt.hardware [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 885.461539] env[64020]: DEBUG nova.virt.hardware [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 885.461689] env[64020]: DEBUG nova.virt.hardware [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 885.461847] env[64020]: DEBUG nova.virt.hardware [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 885.462002] env[64020]: DEBUG nova.virt.hardware [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 885.462182] env[64020]: DEBUG nova.virt.hardware [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 885.463204] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e750a722-9561-41d1-8f9a-823660146024 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.471278] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2020585-ae2f-43af-ae56-6285cc67bbc0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.617518] env[64020]: DEBUG nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 885.619492] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 67da791d3cef47aabad9b0c8deb0f667 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 885.660739] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 67da791d3cef47aabad9b0c8deb0f667 [ 885.672294] env[64020]: DEBUG nova.compute.manager [req-c1fa3a56-d2e4-4578-beb9-6149a3a9b9b0 req-93a02739-844b-459f-a9c0-8af607c4d311 service nova] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Received event network-changed-3c981ae7-e113-416e-a449-8d35634800be {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 885.672509] env[64020]: DEBUG nova.compute.manager [req-c1fa3a56-d2e4-4578-beb9-6149a3a9b9b0 req-93a02739-844b-459f-a9c0-8af607c4d311 service nova] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Refreshing instance network info cache due to event network-changed-3c981ae7-e113-416e-a449-8d35634800be. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 885.672809] env[64020]: DEBUG oslo_concurrency.lockutils [req-c1fa3a56-d2e4-4578-beb9-6149a3a9b9b0 req-93a02739-844b-459f-a9c0-8af607c4d311 service nova] Acquiring lock "refresh_cache-8c4773b7-a8de-4de3-a91f-2252cc34d11b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 885.672962] env[64020]: DEBUG oslo_concurrency.lockutils [req-c1fa3a56-d2e4-4578-beb9-6149a3a9b9b0 req-93a02739-844b-459f-a9c0-8af607c4d311 service nova] Acquired lock "refresh_cache-8c4773b7-a8de-4de3-a91f-2252cc34d11b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 885.673243] env[64020]: DEBUG nova.network.neutron [req-c1fa3a56-d2e4-4578-beb9-6149a3a9b9b0 req-93a02739-844b-459f-a9c0-8af607c4d311 service nova] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Refreshing network info cache for port 3c981ae7-e113-416e-a449-8d35634800be {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 885.673560] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-c1fa3a56-d2e4-4578-beb9-6149a3a9b9b0 req-93a02739-844b-459f-a9c0-8af607c4d311 service nova] Expecting reply to msg df3d8991428a4420956aadfab67749e2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 885.680533] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg df3d8991428a4420956aadfab67749e2 [ 885.795988] env[64020]: ERROR nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3c981ae7-e113-416e-a449-8d35634800be, please check neutron logs for more information. [ 885.795988] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 885.795988] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 885.795988] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 885.795988] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 885.795988] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 885.795988] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 885.795988] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 885.795988] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 885.795988] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 885.795988] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 885.795988] env[64020]: ERROR nova.compute.manager raise self.value [ 885.795988] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 885.795988] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 885.795988] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 885.795988] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 885.796590] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 885.796590] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 885.796590] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3c981ae7-e113-416e-a449-8d35634800be, please check neutron logs for more information. [ 885.796590] env[64020]: ERROR nova.compute.manager [ 885.796590] env[64020]: Traceback (most recent call last): [ 885.796590] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 885.796590] env[64020]: listener.cb(fileno) [ 885.796590] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 885.796590] env[64020]: result = function(*args, **kwargs) [ 885.796590] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 885.796590] env[64020]: return func(*args, **kwargs) [ 885.796590] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 885.796590] env[64020]: raise e [ 885.796590] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 885.796590] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 885.796590] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 885.796590] env[64020]: created_port_ids = self._update_ports_for_instance( [ 885.796590] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 885.796590] env[64020]: with excutils.save_and_reraise_exception(): [ 885.796590] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 885.796590] env[64020]: self.force_reraise() [ 885.796590] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 885.796590] env[64020]: raise self.value [ 885.796590] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 885.796590] env[64020]: updated_port = self._update_port( [ 885.796590] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 885.796590] env[64020]: _ensure_no_port_binding_failure(port) [ 885.796590] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 885.796590] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 885.797325] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 3c981ae7-e113-416e-a449-8d35634800be, please check neutron logs for more information. [ 885.797325] env[64020]: Removing descriptor: 18 [ 885.797325] env[64020]: ERROR nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3c981ae7-e113-416e-a449-8d35634800be, please check neutron logs for more information. [ 885.797325] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Traceback (most recent call last): [ 885.797325] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 885.797325] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] yield resources [ 885.797325] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 885.797325] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] self.driver.spawn(context, instance, image_meta, [ 885.797325] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 885.797325] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 885.797325] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 885.797325] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] vm_ref = self.build_virtual_machine(instance, [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] vif_infos = vmwarevif.get_vif_info(self._session, [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] for vif in network_info: [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] return self._sync_wrapper(fn, *args, **kwargs) [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] self.wait() [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] self[:] = self._gt.wait() [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] return self._exit_event.wait() [ 885.797646] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] result = hub.switch() [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] return self.greenlet.switch() [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] result = function(*args, **kwargs) [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] return func(*args, **kwargs) [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] raise e [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] nwinfo = self.network_api.allocate_for_instance( [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 885.797961] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] created_port_ids = self._update_ports_for_instance( [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] with excutils.save_and_reraise_exception(): [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] self.force_reraise() [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] raise self.value [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] updated_port = self._update_port( [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] _ensure_no_port_binding_failure(port) [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 885.798294] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] raise exception.PortBindingFailed(port_id=port['id']) [ 885.798594] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] nova.exception.PortBindingFailed: Binding failed for port 3c981ae7-e113-416e-a449-8d35634800be, please check neutron logs for more information. [ 885.798594] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] [ 885.798594] env[64020]: INFO nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Terminating instance [ 885.799324] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Acquiring lock "refresh_cache-8c4773b7-a8de-4de3-a91f-2252cc34d11b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 885.883440] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Lock "995288b5-d79a-4af3-a1e8-3571fff2d356" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 22.370s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 885.883827] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8c22eda6-60ff-4309-92a9-af93351d1f9e tempest-ServerDiagnosticsV248Test-1674281418 tempest-ServerDiagnosticsV248Test-1674281418-project-member] Expecting reply to msg bc6539cbef21455aaab07d35807340c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 885.906113] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc6539cbef21455aaab07d35807340c1 [ 886.138299] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1d1466b-3569-4168-827c-df3dff994642 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.141507] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 886.146894] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6b31142-9e6d-41da-a70c-67a11fd26085 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.175599] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58b7e8da-aee4-431c-b40f-63c12d0e21ad {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.185519] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a35c8bfa-336e-469c-b8db-2348ee483ba4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.199061] env[64020]: DEBUG nova.compute.provider_tree [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 886.199637] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 75c6cb955ddd493aac7b88cca662b308 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 886.207888] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 75c6cb955ddd493aac7b88cca662b308 [ 886.298595] env[64020]: DEBUG nova.network.neutron [req-c1fa3a56-d2e4-4578-beb9-6149a3a9b9b0 req-93a02739-844b-459f-a9c0-8af607c4d311 service nova] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 886.377805] env[64020]: DEBUG nova.network.neutron [req-c1fa3a56-d2e4-4578-beb9-6149a3a9b9b0 req-93a02739-844b-459f-a9c0-8af607c4d311 service nova] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 886.378323] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-c1fa3a56-d2e4-4578-beb9-6149a3a9b9b0 req-93a02739-844b-459f-a9c0-8af607c4d311 service nova] Expecting reply to msg a142999f3a1245f9936bc86345dff3d8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 886.386955] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a142999f3a1245f9936bc86345dff3d8 [ 886.572789] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 886.573013] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 886.703020] env[64020]: DEBUG nova.scheduler.client.report [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 886.705330] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 193de097f3234e6d94b7aae163a49abc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 886.716417] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 193de097f3234e6d94b7aae163a49abc [ 886.878958] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquiring lock "3ad11bab-4531-486f-88c0-3ef7153ea0f5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 886.879195] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "3ad11bab-4531-486f-88c0-3ef7153ea0f5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 886.880638] env[64020]: DEBUG oslo_concurrency.lockutils [req-c1fa3a56-d2e4-4578-beb9-6149a3a9b9b0 req-93a02739-844b-459f-a9c0-8af607c4d311 service nova] Releasing lock "refresh_cache-8c4773b7-a8de-4de3-a91f-2252cc34d11b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 886.881182] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Acquired lock "refresh_cache-8c4773b7-a8de-4de3-a91f-2252cc34d11b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 886.881379] env[64020]: DEBUG nova.network.neutron [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 886.881775] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg d88ba9554ff947429a37700cf3717884 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 886.888510] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d88ba9554ff947429a37700cf3717884 [ 887.208759] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.872s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.209317] env[64020]: ERROR nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 73b13abf-97df-4e7d-9625-2864f0a646e9, please check neutron logs for more information. [ 887.209317] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Traceback (most recent call last): [ 887.209317] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 887.209317] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] self.driver.spawn(context, instance, image_meta, [ 887.209317] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 887.209317] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 887.209317] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 887.209317] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] vm_ref = self.build_virtual_machine(instance, [ 887.209317] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 887.209317] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] vif_infos = vmwarevif.get_vif_info(self._session, [ 887.209317] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] for vif in network_info: [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] return self._sync_wrapper(fn, *args, **kwargs) [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] self.wait() [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] self[:] = self._gt.wait() [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] return self._exit_event.wait() [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] current.throw(*self._exc) [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 887.209753] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] result = function(*args, **kwargs) [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] return func(*args, **kwargs) [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] raise e [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] nwinfo = self.network_api.allocate_for_instance( [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] created_port_ids = self._update_ports_for_instance( [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] with excutils.save_and_reraise_exception(): [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] self.force_reraise() [ 887.210224] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 887.210681] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] raise self.value [ 887.210681] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 887.210681] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] updated_port = self._update_port( [ 887.210681] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 887.210681] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] _ensure_no_port_binding_failure(port) [ 887.210681] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 887.210681] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] raise exception.PortBindingFailed(port_id=port['id']) [ 887.210681] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] nova.exception.PortBindingFailed: Binding failed for port 73b13abf-97df-4e7d-9625-2864f0a646e9, please check neutron logs for more information. [ 887.210681] env[64020]: ERROR nova.compute.manager [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] [ 887.210681] env[64020]: DEBUG nova.compute.utils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Binding failed for port 73b13abf-97df-4e7d-9625-2864f0a646e9, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 887.211650] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.239s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 887.213501] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 0f1d70bb090245508927cf915b3d085b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 887.214734] env[64020]: DEBUG nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Build of instance c9a9eccd-4730-4815-8270-8272ac8bdec6 was re-scheduled: Binding failed for port 73b13abf-97df-4e7d-9625-2864f0a646e9, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 887.215154] env[64020]: DEBUG nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 887.215370] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Acquiring lock "refresh_cache-c9a9eccd-4730-4815-8270-8272ac8bdec6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 887.215541] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Acquired lock "refresh_cache-c9a9eccd-4730-4815-8270-8272ac8bdec6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 887.215698] env[64020]: DEBUG nova.network.neutron [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 887.216074] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 3b9549b921404a88b23f572d377c9cdb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 887.222877] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b9549b921404a88b23f572d377c9cdb [ 887.246792] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f1d70bb090245508927cf915b3d085b [ 887.398630] env[64020]: DEBUG nova.network.neutron [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 887.546768] env[64020]: DEBUG nova.network.neutron [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 887.547278] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 83682e2aca2f406098d613935aebc3de in queue reply_57893177120949e6a93cb88e15cd42b4 [ 887.555528] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 83682e2aca2f406098d613935aebc3de [ 887.696873] env[64020]: DEBUG nova.compute.manager [req-da01d39c-a8a7-4f26-9862-7bd6b91f606a req-cf9d3842-51b4-4e21-bf9d-1bbf08c6ee1f service nova] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Received event network-vif-deleted-3c981ae7-e113-416e-a449-8d35634800be {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 887.734475] env[64020]: DEBUG nova.network.neutron [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 887.808704] env[64020]: DEBUG nova.network.neutron [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 887.811754] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg ce8e8e95c9294079903529903869d717 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 887.819879] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce8e8e95c9294079903529903869d717 [ 887.991104] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8804fce-89e5-43c5-955a-c37c1f7a018f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 887.999894] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0ea095a-eff5-40a8-855c-17ef538ac409 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.031616] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02505c3d-0a2d-4fc9-94fd-5620dffa1f52 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.038632] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61238cdc-ab3b-410f-a44f-4d7110cd718c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.051345] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Releasing lock "refresh_cache-8c4773b7-a8de-4de3-a91f-2252cc34d11b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 888.051744] env[64020]: DEBUG nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 888.051933] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 888.052399] env[64020]: DEBUG nova.compute.provider_tree [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 888.052855] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg cfc4b9bd6d3a4f82a1a02e9b5a698a3a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 888.053655] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8a529e33-bdf4-4f96-b3e5-5f6518277da6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.061596] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd126ef2-39fc-4293-acd0-753dc91d5576 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.071451] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfc4b9bd6d3a4f82a1a02e9b5a698a3a [ 888.082240] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 8c4773b7-a8de-4de3-a91f-2252cc34d11b could not be found. [ 888.082453] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 888.082631] env[64020]: INFO nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Took 0.03 seconds to destroy the instance on the hypervisor. [ 888.082864] env[64020]: DEBUG oslo.service.loopingcall [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 888.083079] env[64020]: DEBUG nova.compute.manager [-] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 888.083172] env[64020]: DEBUG nova.network.neutron [-] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 888.097961] env[64020]: DEBUG nova.network.neutron [-] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 888.098444] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bcc87635c0e24e7bb256b58fcff615aa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 888.105753] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bcc87635c0e24e7bb256b58fcff615aa [ 888.311285] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Releasing lock "refresh_cache-c9a9eccd-4730-4815-8270-8272ac8bdec6" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 888.311576] env[64020]: DEBUG nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 888.311759] env[64020]: DEBUG nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 888.311925] env[64020]: DEBUG nova.network.neutron [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 888.326250] env[64020]: DEBUG nova.network.neutron [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 888.326817] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 2b24b5ba527c4f49935211f4f40de51f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 888.333713] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b24b5ba527c4f49935211f4f40de51f [ 888.556924] env[64020]: DEBUG nova.scheduler.client.report [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 888.559562] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 6977defa572b41a7b86353a706e337a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 888.572821] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6977defa572b41a7b86353a706e337a2 [ 888.600485] env[64020]: DEBUG nova.network.neutron [-] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 888.601026] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c689c83ad0f8446786216c5d0964443d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 888.610840] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c689c83ad0f8446786216c5d0964443d [ 888.829375] env[64020]: DEBUG nova.network.neutron [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 888.830012] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg d807875d6df542b1987e2be5f92ac14b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 888.839284] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d807875d6df542b1987e2be5f92ac14b [ 889.062573] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.851s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 889.063209] env[64020]: ERROR nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 842426d0-e222-405d-a943-e75a3af7e696, please check neutron logs for more information. [ 889.063209] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Traceback (most recent call last): [ 889.063209] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 889.063209] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] self.driver.spawn(context, instance, image_meta, [ 889.063209] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 889.063209] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] self._vmops.spawn(context, instance, image_meta, injected_files, [ 889.063209] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 889.063209] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] vm_ref = self.build_virtual_machine(instance, [ 889.063209] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 889.063209] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] vif_infos = vmwarevif.get_vif_info(self._session, [ 889.063209] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] for vif in network_info: [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] return self._sync_wrapper(fn, *args, **kwargs) [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] self.wait() [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] self[:] = self._gt.wait() [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] return self._exit_event.wait() [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] result = hub.switch() [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 889.063552] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] return self.greenlet.switch() [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] result = function(*args, **kwargs) [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] return func(*args, **kwargs) [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] raise e [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] nwinfo = self.network_api.allocate_for_instance( [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] created_port_ids = self._update_ports_for_instance( [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] with excutils.save_and_reraise_exception(): [ 889.063858] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 889.064187] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] self.force_reraise() [ 889.064187] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 889.064187] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] raise self.value [ 889.064187] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 889.064187] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] updated_port = self._update_port( [ 889.064187] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 889.064187] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] _ensure_no_port_binding_failure(port) [ 889.064187] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 889.064187] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] raise exception.PortBindingFailed(port_id=port['id']) [ 889.064187] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] nova.exception.PortBindingFailed: Binding failed for port 842426d0-e222-405d-a943-e75a3af7e696, please check neutron logs for more information. [ 889.064187] env[64020]: ERROR nova.compute.manager [instance: f4682b82-7475-412b-9319-bee1f05a9c63] [ 889.064456] env[64020]: DEBUG nova.compute.utils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Binding failed for port 842426d0-e222-405d-a943-e75a3af7e696, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 889.065544] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.905s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 889.066960] env[64020]: INFO nova.compute.claims [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 889.068519] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 02a83c8e74164671a56e809c6d4656a7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 889.069686] env[64020]: DEBUG nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Build of instance f4682b82-7475-412b-9319-bee1f05a9c63 was re-scheduled: Binding failed for port 842426d0-e222-405d-a943-e75a3af7e696, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 889.070108] env[64020]: DEBUG nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 889.070330] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Acquiring lock "refresh_cache-f4682b82-7475-412b-9319-bee1f05a9c63" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 889.070495] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Acquired lock "refresh_cache-f4682b82-7475-412b-9319-bee1f05a9c63" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 889.070655] env[64020]: DEBUG nova.network.neutron [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 889.070998] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 172d2abe6e7c469194ea5c5269974601 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 889.084133] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 172d2abe6e7c469194ea5c5269974601 [ 889.103590] env[64020]: INFO nova.compute.manager [-] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Took 1.02 seconds to deallocate network for instance. [ 889.105761] env[64020]: DEBUG nova.compute.claims [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 889.105935] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 889.120530] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02a83c8e74164671a56e809c6d4656a7 [ 889.332541] env[64020]: INFO nova.compute.manager [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] [instance: c9a9eccd-4730-4815-8270-8272ac8bdec6] Took 1.02 seconds to deallocate network for instance. [ 889.334200] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 50c144f0278f4cd089fdbd7626e534e7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 889.373558] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50c144f0278f4cd089fdbd7626e534e7 [ 889.574579] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg c9cf1f17693443419046644983bb59b9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 889.582371] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c9cf1f17693443419046644983bb59b9 [ 889.590871] env[64020]: DEBUG nova.network.neutron [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 889.641555] env[64020]: DEBUG nova.network.neutron [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 889.641555] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 649401e864574b0ea1e2fb1a4cbe6b69 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 889.651247] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 649401e864574b0ea1e2fb1a4cbe6b69 [ 889.839162] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 930260d719a74ec1841f4b62fab816b9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 889.873156] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 930260d719a74ec1841f4b62fab816b9 [ 890.143172] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Releasing lock "refresh_cache-f4682b82-7475-412b-9319-bee1f05a9c63" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 890.143410] env[64020]: DEBUG nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 890.143643] env[64020]: DEBUG nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 890.143819] env[64020]: DEBUG nova.network.neutron [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 890.157706] env[64020]: DEBUG nova.network.neutron [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 890.158259] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg fe265d188d174d49bec2648f94cd9650 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 890.165526] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe265d188d174d49bec2648f94cd9650 [ 890.340692] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f703764f-8fc5-4e52-bbd4-734ae4205e16 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 890.350345] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f63d5087-e4f0-44b6-b27a-9e95320cc63f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 890.379752] env[64020]: INFO nova.scheduler.client.report [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Deleted allocations for instance c9a9eccd-4730-4815-8270-8272ac8bdec6 [ 890.385357] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d91ee3fb-3487-4098-8030-695d53c7e5b5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 890.388179] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Expecting reply to msg 3448a37baa3842c5b2d0d5896d197976 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 890.393543] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3ad026a-6d04-4243-b2e5-cfdee31d476d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 890.406862] env[64020]: DEBUG nova.compute.provider_tree [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 890.407355] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 32dbc3471fbc44b699bce93e5c95f159 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 890.409511] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3448a37baa3842c5b2d0d5896d197976 [ 890.417074] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 32dbc3471fbc44b699bce93e5c95f159 [ 890.662213] env[64020]: DEBUG nova.network.neutron [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 890.662799] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 6459fa187f9549b281be9b0ed157d357 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 890.670884] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6459fa187f9549b281be9b0ed157d357 [ 890.890154] env[64020]: DEBUG oslo_concurrency.lockutils [None req-fa573bcb-8285-408b-8539-f0fd602ea1bb tempest-ServerMetadataTestJSON-1783289340 tempest-ServerMetadataTestJSON-1783289340-project-member] Lock "c9a9eccd-4730-4815-8270-8272ac8bdec6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 139.671s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 890.890780] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg e9023f5da0a349f9ba5e89ab941a246a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 890.900964] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9023f5da0a349f9ba5e89ab941a246a [ 890.910512] env[64020]: DEBUG nova.scheduler.client.report [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 890.913379] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg c9b1e26beca845258788c9a51a65e0c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 890.925930] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c9b1e26beca845258788c9a51a65e0c8 [ 891.034232] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "50e9cb9c-10fd-466d-9b11-5175d7955ac9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 891.034474] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "50e9cb9c-10fd-466d-9b11-5175d7955ac9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 891.164659] env[64020]: INFO nova.compute.manager [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] [instance: f4682b82-7475-412b-9319-bee1f05a9c63] Took 1.02 seconds to deallocate network for instance. [ 891.166438] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg deb41a75622f49438b41a1848efe8f86 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 891.200636] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg deb41a75622f49438b41a1848efe8f86 [ 891.393424] env[64020]: DEBUG nova.compute.manager [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 891.395708] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 42fadd55d2284ae6af86d103da629f38 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 891.416011] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.350s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 891.416532] env[64020]: DEBUG nova.compute.manager [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 891.418135] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg ad5e30ad28484e98ac12b8fc7c9c6c2d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 891.419238] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.331s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 891.421096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 4ac4d113bdee4d8cbc5390a6c735aced in queue reply_57893177120949e6a93cb88e15cd42b4 [ 891.438857] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42fadd55d2284ae6af86d103da629f38 [ 891.451116] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad5e30ad28484e98ac12b8fc7c9c6c2d [ 891.456940] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4ac4d113bdee4d8cbc5390a6c735aced [ 891.683936] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 0727f136297b43d2828efada6ea2f473 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 891.718377] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0727f136297b43d2828efada6ea2f473 [ 891.915850] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 891.927010] env[64020]: DEBUG nova.compute.utils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 891.927567] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 165c862d5a9041af9571b4667feac8c3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 891.929421] env[64020]: DEBUG nova.compute.manager [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Not allocating networking since 'none' was specified. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 891.937541] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 165c862d5a9041af9571b4667feac8c3 [ 892.204657] env[64020]: INFO nova.scheduler.client.report [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Deleted allocations for instance f4682b82-7475-412b-9319-bee1f05a9c63 [ 892.210145] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Expecting reply to msg 2d9b5c7b9c32499c98687e7e41ae611c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 892.212720] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b94b08f-fbf1-4881-a2b6-b47e888c187b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.222752] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2937ce58-f453-443c-872b-0d3cc9055c6c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.254939] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d9b5c7b9c32499c98687e7e41ae611c [ 892.255877] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-703c7e5a-3246-4ea6-9f66-1ee624026f97 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.263570] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76800d60-9441-4c78-98ca-cb8f7360bd87 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.278146] env[64020]: DEBUG nova.compute.provider_tree [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 892.278626] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 6fbfe0f0eaf144abb430cc1f33414b32 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 892.286286] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6fbfe0f0eaf144abb430cc1f33414b32 [ 892.429954] env[64020]: DEBUG nova.compute.manager [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 892.431734] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 9f3ec01170304d27b43f5a53615f9c0b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 892.463194] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f3ec01170304d27b43f5a53615f9c0b [ 892.712336] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4c041cc8-8343-4fa1-a8ce-0bf5349cfa49 tempest-ServerExternalEventsTest-295597353 tempest-ServerExternalEventsTest-295597353-project-member] Lock "f4682b82-7475-412b-9319-bee1f05a9c63" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 117.413s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 892.712937] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg c9bcd2cf77bf4a8585b97297f08c76ad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 892.726069] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c9bcd2cf77bf4a8585b97297f08c76ad [ 892.781701] env[64020]: DEBUG nova.scheduler.client.report [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 892.784294] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 221b0dc9de6d45578cfb1827dfc48128 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 892.799725] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 221b0dc9de6d45578cfb1827dfc48128 [ 892.937348] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 903bdd9001474116bd389c1eeb3ddf0d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 892.974440] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 903bdd9001474116bd389c1eeb3ddf0d [ 893.215454] env[64020]: DEBUG nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 893.217803] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 3d76713b4839483c978dfbd9b0e3992b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 893.248275] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d76713b4839483c978dfbd9b0e3992b [ 893.288940] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.870s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 893.289577] env[64020]: ERROR nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b67bb76f-4607-418d-8c89-c6cae26a7466, please check neutron logs for more information. [ 893.289577] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Traceback (most recent call last): [ 893.289577] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 893.289577] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] self.driver.spawn(context, instance, image_meta, [ 893.289577] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 893.289577] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 893.289577] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 893.289577] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] vm_ref = self.build_virtual_machine(instance, [ 893.289577] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 893.289577] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] vif_infos = vmwarevif.get_vif_info(self._session, [ 893.289577] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] for vif in network_info: [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] return self._sync_wrapper(fn, *args, **kwargs) [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] self.wait() [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] self[:] = self._gt.wait() [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] return self._exit_event.wait() [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] current.throw(*self._exc) [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 893.289919] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] result = function(*args, **kwargs) [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] return func(*args, **kwargs) [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] raise e [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] nwinfo = self.network_api.allocate_for_instance( [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] created_port_ids = self._update_ports_for_instance( [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] with excutils.save_and_reraise_exception(): [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] self.force_reraise() [ 893.290282] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 893.290653] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] raise self.value [ 893.290653] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 893.290653] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] updated_port = self._update_port( [ 893.290653] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 893.290653] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] _ensure_no_port_binding_failure(port) [ 893.290653] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 893.290653] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] raise exception.PortBindingFailed(port_id=port['id']) [ 893.290653] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] nova.exception.PortBindingFailed: Binding failed for port b67bb76f-4607-418d-8c89-c6cae26a7466, please check neutron logs for more information. [ 893.290653] env[64020]: ERROR nova.compute.manager [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] [ 893.290653] env[64020]: DEBUG nova.compute.utils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Binding failed for port b67bb76f-4607-418d-8c89-c6cae26a7466, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 893.291481] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.884s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 893.292905] env[64020]: INFO nova.compute.claims [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 893.294439] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 452eac1594c54bb5be96d54d9113650c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 893.295831] env[64020]: DEBUG nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Build of instance 1235c526-3cfb-42b0-8e2d-64cfc59ec35e was re-scheduled: Binding failed for port b67bb76f-4607-418d-8c89-c6cae26a7466, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 893.296250] env[64020]: DEBUG nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 893.296472] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "refresh_cache-1235c526-3cfb-42b0-8e2d-64cfc59ec35e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 893.296616] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquired lock "refresh_cache-1235c526-3cfb-42b0-8e2d-64cfc59ec35e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 893.296770] env[64020]: DEBUG nova.network.neutron [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 893.297111] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg bc45e0ca083d4f9799be35b08b283f21 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 893.303280] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc45e0ca083d4f9799be35b08b283f21 [ 893.330896] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 452eac1594c54bb5be96d54d9113650c [ 893.439675] env[64020]: DEBUG nova.compute.manager [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 893.464685] env[64020]: DEBUG nova.virt.hardware [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 893.464945] env[64020]: DEBUG nova.virt.hardware [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 893.465100] env[64020]: DEBUG nova.virt.hardware [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 893.465277] env[64020]: DEBUG nova.virt.hardware [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 893.465423] env[64020]: DEBUG nova.virt.hardware [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 893.465563] env[64020]: DEBUG nova.virt.hardware [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 893.465766] env[64020]: DEBUG nova.virt.hardware [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 893.465921] env[64020]: DEBUG nova.virt.hardware [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 893.466083] env[64020]: DEBUG nova.virt.hardware [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 893.466241] env[64020]: DEBUG nova.virt.hardware [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 893.466412] env[64020]: DEBUG nova.virt.hardware [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 893.467270] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfaa6c09-91c5-4a46-bed0-e63eee824566 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 893.476166] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-654efdaf-e77e-471f-9897-47edf3bb1670 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 893.489586] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 893.495147] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Creating folder: Project (42c886c4912b4c77a303e084ab4f5bb9). Parent ref: group-v110249. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 893.495406] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-0db5ff2e-3a00-4857-9766-d73f9bb413ca {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 893.505450] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Created folder: Project (42c886c4912b4c77a303e084ab4f5bb9) in parent group-v110249. [ 893.505630] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Creating folder: Instances. Parent ref: group-v110270. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 893.505831] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-23232df1-26f4-46f2-a3cc-1f722611bc12 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 893.513936] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Created folder: Instances in parent group-v110270. [ 893.514150] env[64020]: DEBUG oslo.service.loopingcall [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 893.514323] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 893.514507] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-cbaa14ac-38f7-4453-a870-e6a2fbd0ddd8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 893.529307] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 893.529307] env[64020]: value = "task-407907" [ 893.529307] env[64020]: _type = "Task" [ 893.529307] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 893.535858] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407907, 'name': CreateVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 893.738810] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 893.799902] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 17b28835ea054e5086ae81b3c2a55a0e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 893.807894] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 17b28835ea054e5086ae81b3c2a55a0e [ 893.816943] env[64020]: DEBUG nova.network.neutron [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 893.902757] env[64020]: DEBUG nova.network.neutron [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 893.903261] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 38e90cc0a9f94efda3ff485e1eb38c81 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 893.911993] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 38e90cc0a9f94efda3ff485e1eb38c81 [ 894.040097] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407907, 'name': CreateVM_Task, 'duration_secs': 0.257066} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 894.040286] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 894.040726] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 894.040890] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 894.041260] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 894.041513] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-3c0c86c7-6168-457a-91ad-b28cd608d683 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.046711] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 894.046711] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]522c4b88-ee0e-26fb-a33e-f9a5f1d788cb" [ 894.046711] env[64020]: _type = "Task" [ 894.046711] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 894.054350] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]522c4b88-ee0e-26fb-a33e-f9a5f1d788cb, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 894.409530] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Releasing lock "refresh_cache-1235c526-3cfb-42b0-8e2d-64cfc59ec35e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 894.409774] env[64020]: DEBUG nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 894.409959] env[64020]: DEBUG nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 894.410133] env[64020]: DEBUG nova.network.neutron [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 894.426541] env[64020]: DEBUG nova.network.neutron [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 894.427661] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg a17073cac63b4db98dea71e72ec49339 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 894.434259] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a17073cac63b4db98dea71e72ec49339 [ 894.559111] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]522c4b88-ee0e-26fb-a33e-f9a5f1d788cb, 'name': SearchDatastore_Task, 'duration_secs': 0.008881} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 894.559371] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 894.559596] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 894.559823] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 894.559964] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 894.560174] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 894.560441] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-574dd8ed-7983-48f3-a5cb-dd7134e68dcd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.563333] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e868cb1-637f-4271-87bd-ff5c8f9885ee {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.570638] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e9a3ebd4-5172-4a2f-8324-61e900ba21cc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.574881] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 894.575052] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 894.575972] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f44d0a94-1f6e-4088-907a-b3d4e0a1527b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.602011] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5635881-23dd-4007-9372-1c8993b738e8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.605542] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 894.605542] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52454b04-1152-01bb-f1ee-9dbc9278c634" [ 894.605542] env[64020]: _type = "Task" [ 894.605542] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 894.612047] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-feffaf1c-7ac1-4f21-9693-116b56927975 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.619993] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52454b04-1152-01bb-f1ee-9dbc9278c634, 'name': SearchDatastore_Task, 'duration_secs': 0.008289} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 894.621017] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-fa1aa746-9de3-462a-bc9a-74708937015d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.630255] env[64020]: DEBUG nova.compute.provider_tree [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 894.630732] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 162174c104e84fcd8463b6fabd16170d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 894.634004] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 894.634004] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52f09304-8e3b-4dd9-f08e-9753203d9995" [ 894.634004] env[64020]: _type = "Task" [ 894.634004] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 894.640590] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52f09304-8e3b-4dd9-f08e-9753203d9995, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 894.641095] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 162174c104e84fcd8463b6fabd16170d [ 894.929783] env[64020]: DEBUG nova.network.neutron [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 894.930340] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg f3b76ab17a0d4cb9aaad810c15a99fe3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 894.938998] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3b76ab17a0d4cb9aaad810c15a99fe3 [ 895.134316] env[64020]: DEBUG nova.scheduler.client.report [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 895.136778] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg dfd1f18ef2f94ef58d279ae3e92e3ee8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 895.146987] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52f09304-8e3b-4dd9-f08e-9753203d9995, 'name': SearchDatastore_Task, 'duration_secs': 0.008113} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 895.147417] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 895.147496] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 7ba3e477-a789-4beb-90ac-651520836a5d/7ba3e477-a789-4beb-90ac-651520836a5d.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 895.147964] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-61202fc0-d8cf-4171-bbb2-acf4b145852a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 895.160015] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 895.160015] env[64020]: value = "task-407908" [ 895.160015] env[64020]: _type = "Task" [ 895.160015] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 895.168223] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407908, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 895.183524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dfd1f18ef2f94ef58d279ae3e92e3ee8 [ 895.432966] env[64020]: INFO nova.compute.manager [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: 1235c526-3cfb-42b0-8e2d-64cfc59ec35e] Took 1.02 seconds to deallocate network for instance. [ 895.434970] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 4940d9ce089f413da4cd8494cc205396 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 895.475440] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4940d9ce089f413da4cd8494cc205396 [ 895.639446] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.348s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 895.639978] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 895.641706] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg d543bf49345b4e2a820ed26c66262a1b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 895.642753] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.288s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 895.652021] env[64020]: INFO nova.compute.claims [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 895.652021] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg e5e5ca92a8664ce09d5f4d3101b285f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 895.670399] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407908, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.470712} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 895.670652] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 7ba3e477-a789-4beb-90ac-651520836a5d/7ba3e477-a789-4beb-90ac-651520836a5d.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 895.670857] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 895.671105] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-66d0ce3c-ce05-4a69-b577-741927d01849 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 895.677952] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 895.677952] env[64020]: value = "task-407909" [ 895.677952] env[64020]: _type = "Task" [ 895.677952] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 895.681937] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d543bf49345b4e2a820ed26c66262a1b [ 895.682521] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e5e5ca92a8664ce09d5f4d3101b285f5 [ 895.685714] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407909, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 895.948119] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg db0c327a27af4a7d8447bfea41e731f1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 895.982997] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db0c327a27af4a7d8447bfea41e731f1 [ 896.148667] env[64020]: DEBUG nova.compute.utils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 896.149304] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 042ad071181a447393b7d52684f589e3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 896.151364] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 8f2abd47e4b04a99b1a60519469e5f45 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 896.152358] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 896.152521] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 896.160037] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 042ad071181a447393b7d52684f589e3 [ 896.161952] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f2abd47e4b04a99b1a60519469e5f45 [ 896.188610] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407909, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.057365} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 896.188855] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 896.189611] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6fa8a39a-c3d4-4368-bd99-183cb7f71170 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.194288] env[64020]: DEBUG nova.policy [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6eef8429eed24f899b44e5e605c70791', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a6629d1c8cd94df7a442e558e0ff4f08', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 896.213545] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Reconfiguring VM instance instance-0000003e to attach disk [datastore1] 7ba3e477-a789-4beb-90ac-651520836a5d/7ba3e477-a789-4beb-90ac-651520836a5d.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 896.213824] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-e66bf00f-1d54-4400-8408-955a62d894a4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.233770] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 896.233770] env[64020]: value = "task-407910" [ 896.233770] env[64020]: _type = "Task" [ 896.233770] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 896.241701] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407910, 'name': ReconfigVM_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 896.478923] env[64020]: INFO nova.scheduler.client.report [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Deleted allocations for instance 1235c526-3cfb-42b0-8e2d-64cfc59ec35e [ 896.485160] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 81fa8349ac5c4f8db0491a66f795310c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 896.504896] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81fa8349ac5c4f8db0491a66f795310c [ 896.518020] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Successfully created port: f581214d-ee3d-44b6-b35c-df009ea243c5 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 896.654584] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 896.656270] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg f63b8c3ae24b46d6bc38d108c39c800e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 896.702881] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f63b8c3ae24b46d6bc38d108c39c800e [ 896.747307] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407910, 'name': ReconfigVM_Task, 'duration_secs': 0.268936} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 896.747578] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Reconfigured VM instance instance-0000003e to attach disk [datastore1] 7ba3e477-a789-4beb-90ac-651520836a5d/7ba3e477-a789-4beb-90ac-651520836a5d.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 896.748350] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-4ef25683-943c-40ba-b3dc-d9b4403be9a4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.756271] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 896.756271] env[64020]: value = "task-407911" [ 896.756271] env[64020]: _type = "Task" [ 896.756271] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 896.764152] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407911, 'name': Rename_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 896.967295] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c02ec6eb-b4ae-480e-a400-06738f58fba7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.975765] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a71c008b-f873-4a27-b344-7337c5876921 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.013338] env[64020]: DEBUG oslo_concurrency.lockutils [None req-8b46f4ae-96bc-497d-9d3c-46d61b1b304b tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "1235c526-3cfb-42b0-8e2d-64cfc59ec35e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 117.185s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 897.014171] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg a39f6ee009054da6a02735d6889ec8a5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 897.015740] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-410a6039-ec7c-4ee2-ac25-2b35da2a5bf2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.027418] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a39f6ee009054da6a02735d6889ec8a5 [ 897.028855] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da24f832-5066-426c-8446-1974c43a3878 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.045043] env[64020]: DEBUG nova.compute.provider_tree [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 897.045541] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg dcf96a6012ab4bc7b0c81cc32969e947 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 897.054833] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dcf96a6012ab4bc7b0c81cc32969e947 [ 897.171063] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 809b2dc40def404b9d93a0eacf6b92b9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 897.205490] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 809b2dc40def404b9d93a0eacf6b92b9 [ 897.267766] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407911, 'name': Rename_Task, 'duration_secs': 0.140542} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 897.268879] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 897.268879] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-6f8abe00-8f8b-428d-a521-b4d38bc86529 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.277608] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 897.277608] env[64020]: value = "task-407912" [ 897.277608] env[64020]: _type = "Task" [ 897.277608] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 897.285476] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407912, 'name': PowerOnVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 897.339508] env[64020]: DEBUG nova.compute.manager [req-bed2ff03-e9fc-4b78-a53f-7a73a338e859 req-3b5f106f-57c8-4af6-a4bb-e06c8a444102 service nova] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Received event network-changed-f581214d-ee3d-44b6-b35c-df009ea243c5 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 897.339990] env[64020]: DEBUG nova.compute.manager [req-bed2ff03-e9fc-4b78-a53f-7a73a338e859 req-3b5f106f-57c8-4af6-a4bb-e06c8a444102 service nova] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Refreshing instance network info cache due to event network-changed-f581214d-ee3d-44b6-b35c-df009ea243c5. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 897.340308] env[64020]: DEBUG oslo_concurrency.lockutils [req-bed2ff03-e9fc-4b78-a53f-7a73a338e859 req-3b5f106f-57c8-4af6-a4bb-e06c8a444102 service nova] Acquiring lock "refresh_cache-a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 897.340492] env[64020]: DEBUG oslo_concurrency.lockutils [req-bed2ff03-e9fc-4b78-a53f-7a73a338e859 req-3b5f106f-57c8-4af6-a4bb-e06c8a444102 service nova] Acquired lock "refresh_cache-a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 897.340696] env[64020]: DEBUG nova.network.neutron [req-bed2ff03-e9fc-4b78-a53f-7a73a338e859 req-3b5f106f-57c8-4af6-a4bb-e06c8a444102 service nova] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Refreshing network info cache for port f581214d-ee3d-44b6-b35c-df009ea243c5 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 897.341366] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-bed2ff03-e9fc-4b78-a53f-7a73a338e859 req-3b5f106f-57c8-4af6-a4bb-e06c8a444102 service nova] Expecting reply to msg 87d635c2ad5b4cc6b592f0ed6453ea2b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 897.348253] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 87d635c2ad5b4cc6b592f0ed6453ea2b [ 897.495312] env[64020]: ERROR nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f581214d-ee3d-44b6-b35c-df009ea243c5, please check neutron logs for more information. [ 897.495312] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 897.495312] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 897.495312] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 897.495312] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 897.495312] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 897.495312] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 897.495312] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 897.495312] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 897.495312] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 897.495312] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 897.495312] env[64020]: ERROR nova.compute.manager raise self.value [ 897.495312] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 897.495312] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 897.495312] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 897.495312] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 897.495756] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 897.495756] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 897.495756] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f581214d-ee3d-44b6-b35c-df009ea243c5, please check neutron logs for more information. [ 897.495756] env[64020]: ERROR nova.compute.manager [ 897.495756] env[64020]: Traceback (most recent call last): [ 897.495756] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 897.495756] env[64020]: listener.cb(fileno) [ 897.495756] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 897.495756] env[64020]: result = function(*args, **kwargs) [ 897.495756] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 897.495756] env[64020]: return func(*args, **kwargs) [ 897.495756] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 897.495756] env[64020]: raise e [ 897.495756] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 897.495756] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 897.495756] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 897.495756] env[64020]: created_port_ids = self._update_ports_for_instance( [ 897.495756] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 897.495756] env[64020]: with excutils.save_and_reraise_exception(): [ 897.495756] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 897.495756] env[64020]: self.force_reraise() [ 897.495756] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 897.495756] env[64020]: raise self.value [ 897.495756] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 897.495756] env[64020]: updated_port = self._update_port( [ 897.495756] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 897.495756] env[64020]: _ensure_no_port_binding_failure(port) [ 897.495756] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 897.495756] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 897.496632] env[64020]: nova.exception.PortBindingFailed: Binding failed for port f581214d-ee3d-44b6-b35c-df009ea243c5, please check neutron logs for more information. [ 897.496632] env[64020]: Removing descriptor: 18 [ 897.519212] env[64020]: DEBUG nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 897.521051] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 1ab0c833c97941c6a43e6b8ae1b0ee1d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 897.549457] env[64020]: DEBUG nova.scheduler.client.report [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 897.552034] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg bf3ff532b5c642e493469c9b2897f942 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 897.556745] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ab0c833c97941c6a43e6b8ae1b0ee1d [ 897.567060] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf3ff532b5c642e493469c9b2897f942 [ 897.681322] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 897.707332] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 897.707332] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 897.707332] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 897.707595] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 897.707595] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 897.707595] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 897.707711] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 897.707874] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 897.708300] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 897.708369] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 897.708964] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 897.709425] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-faeb2fc7-e021-4e3f-a058-aee93168d95a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.717149] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd935808-9e75-4d12-ae07-a4cf956b2846 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.730883] env[64020]: ERROR nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f581214d-ee3d-44b6-b35c-df009ea243c5, please check neutron logs for more information. [ 897.730883] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Traceback (most recent call last): [ 897.730883] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 897.730883] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] yield resources [ 897.730883] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 897.730883] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] self.driver.spawn(context, instance, image_meta, [ 897.730883] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 897.730883] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] self._vmops.spawn(context, instance, image_meta, injected_files, [ 897.730883] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 897.730883] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] vm_ref = self.build_virtual_machine(instance, [ 897.730883] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] vif_infos = vmwarevif.get_vif_info(self._session, [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] for vif in network_info: [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] return self._sync_wrapper(fn, *args, **kwargs) [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] self.wait() [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] self[:] = self._gt.wait() [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] return self._exit_event.wait() [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 897.731224] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] current.throw(*self._exc) [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] result = function(*args, **kwargs) [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] return func(*args, **kwargs) [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] raise e [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] nwinfo = self.network_api.allocate_for_instance( [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] created_port_ids = self._update_ports_for_instance( [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] with excutils.save_and_reraise_exception(): [ 897.731586] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 897.731895] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] self.force_reraise() [ 897.731895] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 897.731895] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] raise self.value [ 897.731895] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 897.731895] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] updated_port = self._update_port( [ 897.731895] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 897.731895] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] _ensure_no_port_binding_failure(port) [ 897.731895] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 897.731895] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] raise exception.PortBindingFailed(port_id=port['id']) [ 897.731895] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] nova.exception.PortBindingFailed: Binding failed for port f581214d-ee3d-44b6-b35c-df009ea243c5, please check neutron logs for more information. [ 897.731895] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] [ 897.731895] env[64020]: INFO nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Terminating instance [ 897.733517] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "refresh_cache-a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 897.790779] env[64020]: DEBUG oslo_vmware.api [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407912, 'name': PowerOnVM_Task, 'duration_secs': 0.424604} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 897.791056] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 897.791300] env[64020]: INFO nova.compute.manager [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Took 4.35 seconds to spawn the instance on the hypervisor. [ 897.791488] env[64020]: DEBUG nova.compute.manager [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 897.792372] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5017941d-3bf1-4ebd-8c3e-50194d236cf8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.799769] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 44dadebce4844442af18fd96d601153e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 897.834053] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44dadebce4844442af18fd96d601153e [ 897.858809] env[64020]: DEBUG nova.network.neutron [req-bed2ff03-e9fc-4b78-a53f-7a73a338e859 req-3b5f106f-57c8-4af6-a4bb-e06c8a444102 service nova] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 897.941690] env[64020]: DEBUG nova.network.neutron [req-bed2ff03-e9fc-4b78-a53f-7a73a338e859 req-3b5f106f-57c8-4af6-a4bb-e06c8a444102 service nova] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 897.941690] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-bed2ff03-e9fc-4b78-a53f-7a73a338e859 req-3b5f106f-57c8-4af6-a4bb-e06c8a444102 service nova] Expecting reply to msg b2f4c51b3d244a52b28abd73224e6893 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 897.947843] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b2f4c51b3d244a52b28abd73224e6893 [ 898.040753] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 898.054768] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.412s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 898.055385] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 898.056975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg b133b34aaaef46d5bc1f393e4476aa61 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 898.058009] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.769s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 898.061905] env[64020]: INFO nova.compute.claims [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 898.062592] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg aac8a1db05e8406bbfd41df744bef538 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 898.098450] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b133b34aaaef46d5bc1f393e4476aa61 [ 898.100710] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aac8a1db05e8406bbfd41df744bef538 [ 898.309307] env[64020]: INFO nova.compute.manager [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Took 24.16 seconds to build instance. [ 898.309307] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 7497b786b510473aa42919569ec2ea72 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 898.328492] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7497b786b510473aa42919569ec2ea72 [ 898.443591] env[64020]: DEBUG oslo_concurrency.lockutils [req-bed2ff03-e9fc-4b78-a53f-7a73a338e859 req-3b5f106f-57c8-4af6-a4bb-e06c8a444102 service nova] Releasing lock "refresh_cache-a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 898.443591] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquired lock "refresh_cache-a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 898.443591] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 898.443591] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg ce62333dd7634b66b4da0f559f8f9929 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 898.450128] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce62333dd7634b66b4da0f559f8f9929 [ 898.572173] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg fdc38312d7334f5cabfeda94537802fd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 898.572173] env[64020]: DEBUG nova.compute.utils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 898.572173] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 52835973cd124864ad6f63019bfc48a6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 898.572173] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 898.572173] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 898.574808] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fdc38312d7334f5cabfeda94537802fd [ 898.579636] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52835973cd124864ad6f63019bfc48a6 [ 898.635675] env[64020]: DEBUG nova.policy [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6eef8429eed24f899b44e5e605c70791', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a6629d1c8cd94df7a442e558e0ff4f08', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 898.812655] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9abb0e7b-638f-4e8c-9a01-9c585d7d05e9 tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "7ba3e477-a789-4beb-90ac-651520836a5d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 111.638s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 898.812655] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg ce9e5a200b40491581d76fce77fbd316 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 898.823782] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce9e5a200b40491581d76fce77fbd316 [ 898.947563] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Successfully created port: 8dc219a1-523c-47f8-b1ec-2151f1c76264 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 898.971976] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 899.074301] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 899.076184] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg a6b90b808c564cbdb7645ba93ad6838e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 899.118650] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6b90b808c564cbdb7645ba93ad6838e [ 899.143196] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 2f388e63fd4d4c7da9665e0fd118dca3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 899.144849] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 899.145378] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg a78b965cf18e4d3aa56a98e6b200c3d7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 899.153395] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f388e63fd4d4c7da9665e0fd118dca3 [ 899.153946] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a78b965cf18e4d3aa56a98e6b200c3d7 [ 899.313620] env[64020]: DEBUG nova.compute.manager [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 899.315302] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 4070d0b4b7b84cb7ada56d2f962604a0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 899.347328] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4070d0b4b7b84cb7ada56d2f962604a0 [ 899.350594] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce90511d-f4f4-4968-8dfa-5b2fe64019bf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.360124] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90229bf0-1adc-47ee-aa8e-9afc30dea110 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.397725] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36227107-5b6e-417b-9428-c836c996e201 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.412085] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66d77601-6276-4f9a-9006-27cc659c34c3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.430225] env[64020]: DEBUG nova.compute.provider_tree [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 899.430940] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg f4b6ac037f9b42c0a5b8b67ca7870c07 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 899.439028] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f4b6ac037f9b42c0a5b8b67ca7870c07 [ 899.518965] env[64020]: DEBUG nova.compute.manager [req-4fcd78b9-be95-45e6-a393-dbadf41beb72 req-c03488de-5960-4dde-b681-8d508be40b11 service nova] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Received event network-vif-deleted-f581214d-ee3d-44b6-b35c-df009ea243c5 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 899.581601] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 34571e352e4646a8853cf60e64781e20 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 899.619156] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 34571e352e4646a8853cf60e64781e20 [ 899.647638] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Releasing lock "refresh_cache-a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 899.648675] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 899.649033] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 899.649642] env[64020]: INFO nova.compute.manager [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Rebuilding instance [ 899.653628] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-274b5e0b-8de3-4868-975a-3463aa5ed7de {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.665480] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b96e195b-a56d-4c30-b163-978234f91c01 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.690965] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423 could not be found. [ 899.690965] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 899.690965] env[64020]: INFO nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Took 0.04 seconds to destroy the instance on the hypervisor. [ 899.691164] env[64020]: DEBUG oslo.service.loopingcall [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 899.691544] env[64020]: DEBUG nova.compute.manager [-] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 899.691544] env[64020]: DEBUG nova.network.neutron [-] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 899.704177] env[64020]: DEBUG nova.compute.manager [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 899.705065] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1267dbb5-bf96-4a95-8e10-5b181030e53b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.712924] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 367087f0d8364feaac4ce414339a1743 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 899.714775] env[64020]: DEBUG nova.network.neutron [-] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 899.715249] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg f15e0a38effe4100b490f390a73d45c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 899.727186] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f15e0a38effe4100b490f390a73d45c8 [ 899.745282] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 367087f0d8364feaac4ce414339a1743 [ 899.812414] env[64020]: ERROR nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 8dc219a1-523c-47f8-b1ec-2151f1c76264, please check neutron logs for more information. [ 899.812414] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 899.812414] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 899.812414] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 899.812414] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 899.812414] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 899.812414] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 899.812414] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 899.812414] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 899.812414] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 899.812414] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 899.812414] env[64020]: ERROR nova.compute.manager raise self.value [ 899.812414] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 899.812414] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 899.812414] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 899.812414] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 899.812904] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 899.812904] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 899.812904] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 8dc219a1-523c-47f8-b1ec-2151f1c76264, please check neutron logs for more information. [ 899.812904] env[64020]: ERROR nova.compute.manager [ 899.812904] env[64020]: Traceback (most recent call last): [ 899.812904] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 899.812904] env[64020]: listener.cb(fileno) [ 899.812904] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 899.812904] env[64020]: result = function(*args, **kwargs) [ 899.812904] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 899.812904] env[64020]: return func(*args, **kwargs) [ 899.812904] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 899.812904] env[64020]: raise e [ 899.812904] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 899.812904] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 899.812904] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 899.812904] env[64020]: created_port_ids = self._update_ports_for_instance( [ 899.812904] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 899.812904] env[64020]: with excutils.save_and_reraise_exception(): [ 899.812904] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 899.812904] env[64020]: self.force_reraise() [ 899.812904] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 899.812904] env[64020]: raise self.value [ 899.812904] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 899.812904] env[64020]: updated_port = self._update_port( [ 899.812904] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 899.812904] env[64020]: _ensure_no_port_binding_failure(port) [ 899.812904] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 899.812904] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 899.813730] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 8dc219a1-523c-47f8-b1ec-2151f1c76264, please check neutron logs for more information. [ 899.813730] env[64020]: Removing descriptor: 18 [ 899.830778] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 899.933526] env[64020]: DEBUG nova.scheduler.client.report [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 899.936158] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg f8b30ff36ab14b97a9502fceff997594 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 899.948397] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f8b30ff36ab14b97a9502fceff997594 [ 900.087169] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 900.111396] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 900.111717] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 900.113290] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 900.113560] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 900.113764] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 900.113948] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 900.114971] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 900.115443] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 900.115707] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 900.115938] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 900.116873] env[64020]: DEBUG nova.virt.hardware [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 900.118177] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2ab68c7-b036-4816-86e5-f9e39844ac9f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.126643] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-23e9ea23-879b-4084-9815-4bf0e4c4891e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.142711] env[64020]: ERROR nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 8dc219a1-523c-47f8-b1ec-2151f1c76264, please check neutron logs for more information. [ 900.142711] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Traceback (most recent call last): [ 900.142711] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 900.142711] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] yield resources [ 900.142711] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 900.142711] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] self.driver.spawn(context, instance, image_meta, [ 900.142711] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 900.142711] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 900.142711] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 900.142711] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] vm_ref = self.build_virtual_machine(instance, [ 900.142711] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] vif_infos = vmwarevif.get_vif_info(self._session, [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] for vif in network_info: [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] return self._sync_wrapper(fn, *args, **kwargs) [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] self.wait() [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] self[:] = self._gt.wait() [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] return self._exit_event.wait() [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 900.143369] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] current.throw(*self._exc) [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] result = function(*args, **kwargs) [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] return func(*args, **kwargs) [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] raise e [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] nwinfo = self.network_api.allocate_for_instance( [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] created_port_ids = self._update_ports_for_instance( [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] with excutils.save_and_reraise_exception(): [ 900.143931] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 900.144542] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] self.force_reraise() [ 900.144542] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 900.144542] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] raise self.value [ 900.144542] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 900.144542] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] updated_port = self._update_port( [ 900.144542] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 900.144542] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] _ensure_no_port_binding_failure(port) [ 900.144542] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 900.144542] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] raise exception.PortBindingFailed(port_id=port['id']) [ 900.144542] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] nova.exception.PortBindingFailed: Binding failed for port 8dc219a1-523c-47f8-b1ec-2151f1c76264, please check neutron logs for more information. [ 900.144542] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] [ 900.144542] env[64020]: INFO nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Terminating instance [ 900.145881] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "refresh_cache-f4017c56-63b3-420d-91ab-4565d82305fd" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 900.146037] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquired lock "refresh_cache-f4017c56-63b3-420d-91ab-4565d82305fd" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 900.146237] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 900.146692] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 50925ee2ab304fa9b1260a3de7781c6e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 900.155575] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50925ee2ab304fa9b1260a3de7781c6e [ 900.216727] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 900.216727] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-527a8879-5295-4674-937b-24fef41488ce {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.221273] env[64020]: DEBUG nova.network.neutron [-] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 900.221728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 33f65706948b4f43a1a20327f4046c6c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 900.224511] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 900.224511] env[64020]: value = "task-407913" [ 900.224511] env[64020]: _type = "Task" [ 900.224511] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 900.230406] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33f65706948b4f43a1a20327f4046c6c [ 900.240126] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407913, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 900.440801] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.383s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 900.441407] env[64020]: DEBUG nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 900.443111] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 5db1f79fcd1543a182d01cbc8c8b39d2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 900.444291] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.297s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 900.445552] env[64020]: INFO nova.compute.claims [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 900.447047] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 45952621c0a346aa880cbd603dbadc4e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 900.487606] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5db1f79fcd1543a182d01cbc8c8b39d2 [ 900.491366] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 45952621c0a346aa880cbd603dbadc4e [ 900.666345] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 900.723906] env[64020]: INFO nova.compute.manager [-] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Took 1.03 seconds to deallocate network for instance. [ 900.729025] env[64020]: DEBUG nova.compute.claims [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 900.729194] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 900.735557] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407913, 'name': PowerOffVM_Task, 'duration_secs': 0.152501} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 900.735789] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 900.736015] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 900.736749] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72785f64-b830-4543-b816-d1b3a0b2c815 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.745242] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 900.745478] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-45621068-9f11-4716-95cb-68ba5643d29b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.771871] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 900.772115] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Deleting contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 900.772311] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Deleting the datastore file [datastore1] 7ba3e477-a789-4beb-90ac-651520836a5d {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 900.772561] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-20a77794-6c16-4541-b1ae-4911b992c6d2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.778827] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 900.778827] env[64020]: value = "task-407915" [ 900.778827] env[64020]: _type = "Task" [ 900.778827] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 900.792955] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407915, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 900.855088] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 900.855610] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 0fdcf4d7becd45698161fcb7bb5f3d6d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 900.898381] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0fdcf4d7becd45698161fcb7bb5f3d6d [ 900.949594] env[64020]: DEBUG nova.compute.utils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 900.950312] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 0a4cedecd36a42ccbcd7a43efab297bf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 900.955192] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg b108538f6e5945cf83b53562121efd19 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 900.960022] env[64020]: DEBUG nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 900.960022] env[64020]: DEBUG nova.network.neutron [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 900.960022] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "bf317705-3bf8-4522-b41a-fc023bf766d0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 900.960022] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "bf317705-3bf8-4522-b41a-fc023bf766d0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 900.961072] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a4cedecd36a42ccbcd7a43efab297bf [ 900.963421] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b108538f6e5945cf83b53562121efd19 [ 901.026085] env[64020]: DEBUG nova.policy [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f15e4807c6834d978f0587e6a0c294ad', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'de42ea1314dd4f3ba43cedb2ada7fd64', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 901.293829] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407915, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.104197} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 901.293829] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 901.293829] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Deleted contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 901.293829] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 901.293829] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 18d5fe5aff0441ed98bcdb6459ae466a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 901.328089] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18d5fe5aff0441ed98bcdb6459ae466a [ 901.357743] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Releasing lock "refresh_cache-f4017c56-63b3-420d-91ab-4565d82305fd" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 901.358161] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 901.358339] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 901.358625] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4f24cf45-3585-4e9c-ab4e-b2f8d21e4344 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.367678] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-142e80b4-e48f-4e91-8797-ebd7b7c827e9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.389782] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f4017c56-63b3-420d-91ab-4565d82305fd could not be found. [ 901.390046] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 901.390218] env[64020]: INFO nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Took 0.03 seconds to destroy the instance on the hypervisor. [ 901.390470] env[64020]: DEBUG oslo.service.loopingcall [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 901.390687] env[64020]: DEBUG nova.compute.manager [-] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 901.390775] env[64020]: DEBUG nova.network.neutron [-] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 901.425630] env[64020]: DEBUG nova.network.neutron [-] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 901.425630] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b3f1078f8ba14f868b47171758ea1f1f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 901.425630] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3f1078f8ba14f868b47171758ea1f1f [ 901.457512] env[64020]: DEBUG nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 901.459607] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 93f842596e2845caa76d08cf56a64f08 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 901.490698] env[64020]: DEBUG nova.network.neutron [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Successfully created port: a7146506-944d-49f6-bbee-01d781fb3c39 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 901.495054] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93f842596e2845caa76d08cf56a64f08 [ 901.667464] env[64020]: DEBUG nova.compute.manager [req-5130dcd2-a897-4f6b-824b-52b09882eda4 req-f1b6915e-2055-4811-8f97-8fecbb87a4e0 service nova] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Received event network-changed-8dc219a1-523c-47f8-b1ec-2151f1c76264 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 901.667802] env[64020]: DEBUG nova.compute.manager [req-5130dcd2-a897-4f6b-824b-52b09882eda4 req-f1b6915e-2055-4811-8f97-8fecbb87a4e0 service nova] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Refreshing instance network info cache due to event network-changed-8dc219a1-523c-47f8-b1ec-2151f1c76264. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 901.668221] env[64020]: DEBUG oslo_concurrency.lockutils [req-5130dcd2-a897-4f6b-824b-52b09882eda4 req-f1b6915e-2055-4811-8f97-8fecbb87a4e0 service nova] Acquiring lock "refresh_cache-f4017c56-63b3-420d-91ab-4565d82305fd" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 901.668466] env[64020]: DEBUG oslo_concurrency.lockutils [req-5130dcd2-a897-4f6b-824b-52b09882eda4 req-f1b6915e-2055-4811-8f97-8fecbb87a4e0 service nova] Acquired lock "refresh_cache-f4017c56-63b3-420d-91ab-4565d82305fd" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 901.668736] env[64020]: DEBUG nova.network.neutron [req-5130dcd2-a897-4f6b-824b-52b09882eda4 req-f1b6915e-2055-4811-8f97-8fecbb87a4e0 service nova] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Refreshing network info cache for port 8dc219a1-523c-47f8-b1ec-2151f1c76264 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 901.669262] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-5130dcd2-a897-4f6b-824b-52b09882eda4 req-f1b6915e-2055-4811-8f97-8fecbb87a4e0 service nova] Expecting reply to msg 292cdfc2c7c049798c438201ae2f09bf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 901.675298] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 292cdfc2c7c049798c438201ae2f09bf [ 901.798497] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 08b45f4d01ee4ce0938f3dbb8599b013 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 901.819305] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a972cc62-93c5-4955-9ad8-338528653bbe {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.827079] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b1277ea-6979-424f-b06e-3066ec587edd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.831804] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08b45f4d01ee4ce0938f3dbb8599b013 [ 901.865477] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9ea7518-1f3f-48ad-bec2-750192b843ed {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.875775] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4bc1035e-5eb4-48fb-a5dc-77fdb32e2bae {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.890414] env[64020]: DEBUG nova.compute.provider_tree [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 901.890907] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg cdebc192c1c64be4b1377fef733d11d1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 901.898043] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cdebc192c1c64be4b1377fef733d11d1 [ 901.911534] env[64020]: DEBUG nova.network.neutron [-] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 901.911862] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 031bcc9906194aa8a0a867b24b90415d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 901.920341] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 031bcc9906194aa8a0a867b24b90415d [ 901.966542] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 82c4cebe31334d94acf09d3769cae82f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 902.003977] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82c4cebe31334d94acf09d3769cae82f [ 902.196554] env[64020]: DEBUG nova.network.neutron [req-5130dcd2-a897-4f6b-824b-52b09882eda4 req-f1b6915e-2055-4811-8f97-8fecbb87a4e0 service nova] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 902.315225] env[64020]: DEBUG nova.network.neutron [req-5130dcd2-a897-4f6b-824b-52b09882eda4 req-f1b6915e-2055-4811-8f97-8fecbb87a4e0 service nova] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 902.316392] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-5130dcd2-a897-4f6b-824b-52b09882eda4 req-f1b6915e-2055-4811-8f97-8fecbb87a4e0 service nova] Expecting reply to msg c12cc95c9d034ab7a8294a65a6ce536c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 902.326264] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c12cc95c9d034ab7a8294a65a6ce536c [ 902.329418] env[64020]: ERROR nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a7146506-944d-49f6-bbee-01d781fb3c39, please check neutron logs for more information. [ 902.329418] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 902.329418] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 902.329418] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 902.329418] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 902.329418] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 902.329418] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 902.329418] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 902.329418] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 902.329418] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 902.329418] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 902.329418] env[64020]: ERROR nova.compute.manager raise self.value [ 902.329418] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 902.329418] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 902.329418] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 902.329418] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 902.329839] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 902.329839] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 902.329839] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a7146506-944d-49f6-bbee-01d781fb3c39, please check neutron logs for more information. [ 902.329839] env[64020]: ERROR nova.compute.manager [ 902.329839] env[64020]: Traceback (most recent call last): [ 902.329839] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 902.329839] env[64020]: listener.cb(fileno) [ 902.329839] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 902.329839] env[64020]: result = function(*args, **kwargs) [ 902.329839] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 902.329839] env[64020]: return func(*args, **kwargs) [ 902.329839] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 902.329839] env[64020]: raise e [ 902.329839] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 902.329839] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 902.329839] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 902.329839] env[64020]: created_port_ids = self._update_ports_for_instance( [ 902.329839] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 902.329839] env[64020]: with excutils.save_and_reraise_exception(): [ 902.329839] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 902.329839] env[64020]: self.force_reraise() [ 902.329839] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 902.329839] env[64020]: raise self.value [ 902.329839] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 902.329839] env[64020]: updated_port = self._update_port( [ 902.329839] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 902.329839] env[64020]: _ensure_no_port_binding_failure(port) [ 902.329839] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 902.329839] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 902.330546] env[64020]: nova.exception.PortBindingFailed: Binding failed for port a7146506-944d-49f6-bbee-01d781fb3c39, please check neutron logs for more information. [ 902.330546] env[64020]: Removing descriptor: 18 [ 902.331340] env[64020]: DEBUG nova.virt.hardware [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 902.331571] env[64020]: DEBUG nova.virt.hardware [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 902.331722] env[64020]: DEBUG nova.virt.hardware [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 902.331895] env[64020]: DEBUG nova.virt.hardware [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 902.332062] env[64020]: DEBUG nova.virt.hardware [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 902.332245] env[64020]: DEBUG nova.virt.hardware [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 902.332471] env[64020]: DEBUG nova.virt.hardware [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 902.332658] env[64020]: DEBUG nova.virt.hardware [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 902.332870] env[64020]: DEBUG nova.virt.hardware [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 902.333045] env[64020]: DEBUG nova.virt.hardware [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 902.333213] env[64020]: DEBUG nova.virt.hardware [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 902.334225] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3a4415d-e871-490f-9674-8addbf610c2d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.342598] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ada8ef7-6bbf-4003-8655-5a0f6868b1ee {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.356804] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 902.362257] env[64020]: DEBUG oslo.service.loopingcall [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 902.362502] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 902.362706] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-7ad440a3-f7a4-43cf-a5d7-2c981f40b877 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.378976] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 902.378976] env[64020]: value = "task-407916" [ 902.378976] env[64020]: _type = "Task" [ 902.378976] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 902.386322] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407916, 'name': CreateVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 902.393239] env[64020]: DEBUG nova.scheduler.client.report [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 902.396842] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 94ca2de506f84ce393b921f6b9995861 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 902.421392] env[64020]: INFO nova.compute.manager [-] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Took 1.03 seconds to deallocate network for instance. [ 902.422424] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94ca2de506f84ce393b921f6b9995861 [ 902.425323] env[64020]: DEBUG nova.compute.claims [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 902.425496] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 902.470254] env[64020]: DEBUG nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 902.493727] env[64020]: DEBUG nova.virt.hardware [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 902.494025] env[64020]: DEBUG nova.virt.hardware [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 902.494180] env[64020]: DEBUG nova.virt.hardware [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 902.494371] env[64020]: DEBUG nova.virt.hardware [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 902.494509] env[64020]: DEBUG nova.virt.hardware [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 902.494647] env[64020]: DEBUG nova.virt.hardware [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 902.494854] env[64020]: DEBUG nova.virt.hardware [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 902.495035] env[64020]: DEBUG nova.virt.hardware [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 902.495293] env[64020]: DEBUG nova.virt.hardware [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 902.495464] env[64020]: DEBUG nova.virt.hardware [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 902.495646] env[64020]: DEBUG nova.virt.hardware [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 902.496886] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b9a04ac-7525-4f03-a12d-93032d756ffa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.505673] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-554ff348-b975-4e0d-a145-f8b54e7650f5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.523329] env[64020]: ERROR nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a7146506-944d-49f6-bbee-01d781fb3c39, please check neutron logs for more information. [ 902.523329] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Traceback (most recent call last): [ 902.523329] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 902.523329] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] yield resources [ 902.523329] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 902.523329] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] self.driver.spawn(context, instance, image_meta, [ 902.523329] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 902.523329] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 902.523329] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 902.523329] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] vm_ref = self.build_virtual_machine(instance, [ 902.523329] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] vif_infos = vmwarevif.get_vif_info(self._session, [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] for vif in network_info: [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] return self._sync_wrapper(fn, *args, **kwargs) [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] self.wait() [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] self[:] = self._gt.wait() [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] return self._exit_event.wait() [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 902.523681] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] current.throw(*self._exc) [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] result = function(*args, **kwargs) [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] return func(*args, **kwargs) [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] raise e [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] nwinfo = self.network_api.allocate_for_instance( [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] created_port_ids = self._update_ports_for_instance( [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] with excutils.save_and_reraise_exception(): [ 902.524019] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 902.524337] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] self.force_reraise() [ 902.524337] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 902.524337] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] raise self.value [ 902.524337] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 902.524337] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] updated_port = self._update_port( [ 902.524337] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 902.524337] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] _ensure_no_port_binding_failure(port) [ 902.524337] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 902.524337] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] raise exception.PortBindingFailed(port_id=port['id']) [ 902.524337] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] nova.exception.PortBindingFailed: Binding failed for port a7146506-944d-49f6-bbee-01d781fb3c39, please check neutron logs for more information. [ 902.524337] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] [ 902.524337] env[64020]: INFO nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Terminating instance [ 902.525884] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Acquiring lock "refresh_cache-5a021861-9784-431d-b717-c7b24fe8525c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 902.526433] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Acquired lock "refresh_cache-5a021861-9784-431d-b717-c7b24fe8525c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 902.526613] env[64020]: DEBUG nova.network.neutron [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 902.527042] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg cfaea665ff7e4fa3a58674ea4c9d26a9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 902.535389] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfaea665ff7e4fa3a58674ea4c9d26a9 [ 902.799939] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 902.800236] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 902.800887] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 5e95f6d31bd546408ab39892934b4c59 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 902.815890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e95f6d31bd546408ab39892934b4c59 [ 902.818946] env[64020]: DEBUG oslo_concurrency.lockutils [req-5130dcd2-a897-4f6b-824b-52b09882eda4 req-f1b6915e-2055-4811-8f97-8fecbb87a4e0 service nova] Releasing lock "refresh_cache-f4017c56-63b3-420d-91ab-4565d82305fd" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 902.819182] env[64020]: DEBUG nova.compute.manager [req-5130dcd2-a897-4f6b-824b-52b09882eda4 req-f1b6915e-2055-4811-8f97-8fecbb87a4e0 service nova] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Received event network-vif-deleted-8dc219a1-523c-47f8-b1ec-2151f1c76264 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 902.889501] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407916, 'name': CreateVM_Task, 'duration_secs': 0.285245} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 902.889666] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 902.890068] env[64020]: DEBUG oslo_concurrency.lockutils [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 902.890215] env[64020]: DEBUG oslo_concurrency.lockutils [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 902.890518] env[64020]: DEBUG oslo_concurrency.lockutils [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 902.890758] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-377ef067-f2f1-445a-8c30-ae64c00f4f95 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.895354] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 902.895354] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]523431d2-c307-d721-6066-773c33abbde1" [ 902.895354] env[64020]: _type = "Task" [ 902.895354] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 902.904871] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]523431d2-c307-d721-6066-773c33abbde1, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 902.907658] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.463s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 902.908152] env[64020]: DEBUG nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 902.909767] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 05cdac915ee74c478481c721a29ad61b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 902.911340] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.770s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 902.913202] env[64020]: INFO nova.compute.claims [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 902.914463] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg d3f4982224ed42288ec51a4e4c417a61 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 902.953090] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d3f4982224ed42288ec51a4e4c417a61 [ 902.953958] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05cdac915ee74c478481c721a29ad61b [ 903.043999] env[64020]: DEBUG nova.network.neutron [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 903.192026] env[64020]: DEBUG nova.network.neutron [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 903.192586] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 93c4a95d04b84da1ab1ec2c9e9b200c2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 903.201073] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93c4a95d04b84da1ab1ec2c9e9b200c2 [ 903.305737] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 903.305925] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Starting heal instance info cache {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 903.306044] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Rebuilding the list of instances to heal {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 903.306627] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 601c77dd7be6435ca0a5ad8b3a850495 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 903.326365] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 601c77dd7be6435ca0a5ad8b3a850495 [ 903.407253] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]523431d2-c307-d721-6066-773c33abbde1, 'name': SearchDatastore_Task, 'duration_secs': 0.009636} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 903.407592] env[64020]: DEBUG oslo_concurrency.lockutils [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 903.407825] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 903.408075] env[64020]: DEBUG oslo_concurrency.lockutils [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 903.408226] env[64020]: DEBUG oslo_concurrency.lockutils [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 903.408407] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 903.408671] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-82d874be-b3cf-4679-8ecb-1b9283a996aa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.413233] env[64020]: DEBUG nova.compute.utils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 903.413906] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 56cc3b0dcc9c4bbe89c244504096295c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 903.424245] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 56cc3b0dcc9c4bbe89c244504096295c [ 903.426079] env[64020]: DEBUG nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 903.426252] env[64020]: DEBUG nova.network.neutron [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 903.429669] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 803f7cdf05384abebd8765df1c6d1709 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 903.431337] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 903.431522] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 903.432717] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c6cd7ad1-0d15-4c75-b77f-b791b755b87a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.437425] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 803f7cdf05384abebd8765df1c6d1709 [ 903.439381] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 903.439381] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]528b4cbe-74d0-be72-7f1d-a8f918c92ffd" [ 903.439381] env[64020]: _type = "Task" [ 903.439381] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 903.448526] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]528b4cbe-74d0-be72-7f1d-a8f918c92ffd, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 903.532393] env[64020]: DEBUG nova.policy [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fd08d772c4dd4bf5986d37b8474d30a1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '540a31f288b54221a4d0348b147e40e7', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 903.695859] env[64020]: DEBUG nova.compute.manager [req-efcc8215-a423-4cb1-99c2-3e85059261aa req-cc953e35-5473-48ef-a26d-282991aec380 service nova] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Received event network-changed-a7146506-944d-49f6-bbee-01d781fb3c39 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 903.696067] env[64020]: DEBUG nova.compute.manager [req-efcc8215-a423-4cb1-99c2-3e85059261aa req-cc953e35-5473-48ef-a26d-282991aec380 service nova] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Refreshing instance network info cache due to event network-changed-a7146506-944d-49f6-bbee-01d781fb3c39. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 903.696260] env[64020]: DEBUG oslo_concurrency.lockutils [req-efcc8215-a423-4cb1-99c2-3e85059261aa req-cc953e35-5473-48ef-a26d-282991aec380 service nova] Acquiring lock "refresh_cache-5a021861-9784-431d-b717-c7b24fe8525c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 903.699093] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Releasing lock "refresh_cache-5a021861-9784-431d-b717-c7b24fe8525c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 903.699451] env[64020]: DEBUG nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 903.699630] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 903.700197] env[64020]: DEBUG oslo_concurrency.lockutils [req-efcc8215-a423-4cb1-99c2-3e85059261aa req-cc953e35-5473-48ef-a26d-282991aec380 service nova] Acquired lock "refresh_cache-5a021861-9784-431d-b717-c7b24fe8525c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 903.700362] env[64020]: DEBUG nova.network.neutron [req-efcc8215-a423-4cb1-99c2-3e85059261aa req-cc953e35-5473-48ef-a26d-282991aec380 service nova] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Refreshing network info cache for port a7146506-944d-49f6-bbee-01d781fb3c39 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 903.700769] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-efcc8215-a423-4cb1-99c2-3e85059261aa req-cc953e35-5473-48ef-a26d-282991aec380 service nova] Expecting reply to msg f42853ab4c8d41cf99338e38b2a70858 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 903.702128] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-afe2d725-1e34-423b-9760-d797dfbf98df {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.711740] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f42853ab4c8d41cf99338e38b2a70858 [ 903.716025] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e432bc76-56c8-4bbc-9297-15c713235505 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.738560] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5a021861-9784-431d-b717-c7b24fe8525c could not be found. [ 903.738780] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 903.738951] env[64020]: INFO nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Took 0.04 seconds to destroy the instance on the hypervisor. [ 903.739220] env[64020]: DEBUG oslo.service.loopingcall [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 903.739446] env[64020]: DEBUG nova.compute.manager [-] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 903.739538] env[64020]: DEBUG nova.network.neutron [-] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 903.753419] env[64020]: DEBUG nova.network.neutron [-] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 903.753924] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg edeac5a1cf9840c58dbc6064f78f698d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 903.761088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg edeac5a1cf9840c58dbc6064f78f698d [ 903.810793] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 903.811031] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 903.811101] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 903.811275] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 903.811542] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 903.811542] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 903.811725] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "refresh_cache-7ba3e477-a789-4beb-90ac-651520836a5d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 903.811839] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquired lock "refresh_cache-7ba3e477-a789-4beb-90ac-651520836a5d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 903.811966] env[64020]: DEBUG nova.network.neutron [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Forcefully refreshing network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2004}} [ 903.812126] env[64020]: DEBUG nova.objects.instance [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lazy-loading 'info_cache' on Instance uuid 7ba3e477-a789-4beb-90ac-651520836a5d {{(pid=64020) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 903.812715] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg c3cf09b693c043708e9b0e735b019bda in queue reply_57893177120949e6a93cb88e15cd42b4 [ 903.823648] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3cf09b693c043708e9b0e735b019bda [ 903.927308] env[64020]: DEBUG nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 903.927308] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg f87d2da2e35f4432b4330867c5cf0703 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 903.950972] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]528b4cbe-74d0-be72-7f1d-a8f918c92ffd, 'name': SearchDatastore_Task, 'duration_secs': 0.008554} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 903.952364] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-062b12d1-1049-4900-bef5-bf28eb11f9cf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.961029] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 903.961029] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52a1b6a9-74c6-c63d-30d4-cc3e80730b20" [ 903.961029] env[64020]: _type = "Task" [ 903.961029] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 903.969767] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52a1b6a9-74c6-c63d-30d4-cc3e80730b20, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 903.979167] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f87d2da2e35f4432b4330867c5cf0703 [ 904.218600] env[64020]: DEBUG nova.network.neutron [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Successfully created port: 2541ed1d-017e-49b6-b5c7-4737c699f975 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 904.241948] env[64020]: DEBUG nova.network.neutron [req-efcc8215-a423-4cb1-99c2-3e85059261aa req-cc953e35-5473-48ef-a26d-282991aec380 service nova] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 904.256271] env[64020]: DEBUG nova.network.neutron [-] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 904.256715] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c0b79beb6e014095a6708ea8cd56a73c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 904.263575] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66d43337-fc74-4446-abc5-8d5c58dd81ac {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.266652] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0b79beb6e014095a6708ea8cd56a73c [ 904.271958] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cde754f1-04c1-4071-9ea4-909c35100843 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.304624] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc534c5a-724d-4610-8234-e8824e30da64 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.311800] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-670ab62d-f2f7-4ae1-bbb3-abf5d32b54b9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.325630] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 19fa43d58141474da7c24df64e52c001 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 904.326854] env[64020]: DEBUG nova.compute.provider_tree [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 904.327294] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 3cf79dc04c8e43a4bbeda227c789bedb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 904.334688] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3cf79dc04c8e43a4bbeda227c789bedb [ 904.335134] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19fa43d58141474da7c24df64e52c001 [ 904.357154] env[64020]: DEBUG nova.network.neutron [req-efcc8215-a423-4cb1-99c2-3e85059261aa req-cc953e35-5473-48ef-a26d-282991aec380 service nova] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 904.357645] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-efcc8215-a423-4cb1-99c2-3e85059261aa req-cc953e35-5473-48ef-a26d-282991aec380 service nova] Expecting reply to msg bb90e556598f426e842fbcb580a4b9a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 904.365100] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb90e556598f426e842fbcb580a4b9a2 [ 904.430716] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg e9ad62b81b4f4ca0ab74da33e074f295 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 904.467105] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9ad62b81b4f4ca0ab74da33e074f295 [ 904.476387] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52a1b6a9-74c6-c63d-30d4-cc3e80730b20, 'name': SearchDatastore_Task, 'duration_secs': 0.009053} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 904.476387] env[64020]: DEBUG oslo_concurrency.lockutils [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 904.476387] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 7ba3e477-a789-4beb-90ac-651520836a5d/7ba3e477-a789-4beb-90ac-651520836a5d.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 904.476387] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-02bdbba3-b40c-4d13-8174-01831e84c747 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.482271] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 904.482271] env[64020]: value = "task-407917" [ 904.482271] env[64020]: _type = "Task" [ 904.482271] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 904.490215] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407917, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 904.758877] env[64020]: INFO nova.compute.manager [-] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Took 1.02 seconds to deallocate network for instance. [ 904.763467] env[64020]: DEBUG nova.compute.claims [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 904.763670] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 904.832044] env[64020]: DEBUG nova.scheduler.client.report [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 904.834539] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 15fd9ab442f44131b73dce24e8134845 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 904.848264] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 15fd9ab442f44131b73dce24e8134845 [ 904.849553] env[64020]: DEBUG nova.network.neutron [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 904.849891] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg ed60b20b486744c9a6c71b1253aba450 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 904.859231] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed60b20b486744c9a6c71b1253aba450 [ 904.859857] env[64020]: DEBUG oslo_concurrency.lockutils [req-efcc8215-a423-4cb1-99c2-3e85059261aa req-cc953e35-5473-48ef-a26d-282991aec380 service nova] Releasing lock "refresh_cache-5a021861-9784-431d-b717-c7b24fe8525c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 904.860115] env[64020]: DEBUG nova.compute.manager [req-efcc8215-a423-4cb1-99c2-3e85059261aa req-cc953e35-5473-48ef-a26d-282991aec380 service nova] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Received event network-vif-deleted-a7146506-944d-49f6-bbee-01d781fb3c39 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 904.933858] env[64020]: DEBUG nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 904.958376] env[64020]: DEBUG nova.virt.hardware [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 904.958622] env[64020]: DEBUG nova.virt.hardware [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 904.958766] env[64020]: DEBUG nova.virt.hardware [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 904.958936] env[64020]: DEBUG nova.virt.hardware [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 904.959143] env[64020]: DEBUG nova.virt.hardware [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 904.959326] env[64020]: DEBUG nova.virt.hardware [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 904.959542] env[64020]: DEBUG nova.virt.hardware [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 904.959690] env[64020]: DEBUG nova.virt.hardware [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 904.960053] env[64020]: DEBUG nova.virt.hardware [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 904.960264] env[64020]: DEBUG nova.virt.hardware [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 904.960440] env[64020]: DEBUG nova.virt.hardware [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 904.961338] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af4e92fa-a974-4c67-abb0-cb538ab9623b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.969871] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c12f52ca-d71d-458a-a594-d484d1a0cd12 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.989755] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407917, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.485994} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 904.989996] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 7ba3e477-a789-4beb-90ac-651520836a5d/7ba3e477-a789-4beb-90ac-651520836a5d.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 904.990201] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 904.990439] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-8ce04842-9301-4db5-a64b-27cb65174786 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.998429] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 904.998429] env[64020]: value = "task-407918" [ 904.998429] env[64020]: _type = "Task" [ 904.998429] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 905.003612] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407918, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 905.337104] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.425s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 905.337475] env[64020]: DEBUG nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 905.339171] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg c2114e172d1a450795d166f2b6dfd847 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 905.340299] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.234s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 905.342022] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg d4da2b075bf24ed9be4a915b145fecdc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 905.343882] env[64020]: ERROR nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 2541ed1d-017e-49b6-b5c7-4737c699f975, please check neutron logs for more information. [ 905.343882] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 905.343882] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 905.343882] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 905.343882] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 905.343882] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 905.343882] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 905.343882] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 905.343882] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 905.343882] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 905.343882] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 905.343882] env[64020]: ERROR nova.compute.manager raise self.value [ 905.343882] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 905.343882] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 905.343882] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 905.343882] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 905.344437] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 905.344437] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 905.344437] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 2541ed1d-017e-49b6-b5c7-4737c699f975, please check neutron logs for more information. [ 905.344437] env[64020]: ERROR nova.compute.manager [ 905.344437] env[64020]: Traceback (most recent call last): [ 905.344437] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 905.344437] env[64020]: listener.cb(fileno) [ 905.344437] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 905.344437] env[64020]: result = function(*args, **kwargs) [ 905.344437] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 905.344437] env[64020]: return func(*args, **kwargs) [ 905.344437] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 905.344437] env[64020]: raise e [ 905.344437] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 905.344437] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 905.344437] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 905.344437] env[64020]: created_port_ids = self._update_ports_for_instance( [ 905.344437] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 905.344437] env[64020]: with excutils.save_and_reraise_exception(): [ 905.344437] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 905.344437] env[64020]: self.force_reraise() [ 905.344437] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 905.344437] env[64020]: raise self.value [ 905.344437] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 905.344437] env[64020]: updated_port = self._update_port( [ 905.344437] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 905.344437] env[64020]: _ensure_no_port_binding_failure(port) [ 905.344437] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 905.344437] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 905.345254] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 2541ed1d-017e-49b6-b5c7-4737c699f975, please check neutron logs for more information. [ 905.345254] env[64020]: Removing descriptor: 18 [ 905.345254] env[64020]: ERROR nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 2541ed1d-017e-49b6-b5c7-4737c699f975, please check neutron logs for more information. [ 905.345254] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Traceback (most recent call last): [ 905.345254] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 905.345254] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] yield resources [ 905.345254] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 905.345254] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] self.driver.spawn(context, instance, image_meta, [ 905.345254] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 905.345254] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 905.345254] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 905.345254] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] vm_ref = self.build_virtual_machine(instance, [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] vif_infos = vmwarevif.get_vif_info(self._session, [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] for vif in network_info: [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] return self._sync_wrapper(fn, *args, **kwargs) [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] self.wait() [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] self[:] = self._gt.wait() [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] return self._exit_event.wait() [ 905.345569] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] result = hub.switch() [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] return self.greenlet.switch() [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] result = function(*args, **kwargs) [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] return func(*args, **kwargs) [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] raise e [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] nwinfo = self.network_api.allocate_for_instance( [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 905.345919] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] created_port_ids = self._update_ports_for_instance( [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] with excutils.save_and_reraise_exception(): [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] self.force_reraise() [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] raise self.value [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] updated_port = self._update_port( [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] _ensure_no_port_binding_failure(port) [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 905.346228] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] raise exception.PortBindingFailed(port_id=port['id']) [ 905.346519] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] nova.exception.PortBindingFailed: Binding failed for port 2541ed1d-017e-49b6-b5c7-4737c699f975, please check neutron logs for more information. [ 905.346519] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] [ 905.346519] env[64020]: INFO nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Terminating instance [ 905.354023] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "refresh_cache-414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 905.354023] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquired lock "refresh_cache-414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 905.354135] env[64020]: DEBUG nova.network.neutron [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 905.354580] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg d7033893fd3347dc8c6dbce41749e241 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 905.371922] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7033893fd3347dc8c6dbce41749e241 [ 905.378963] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2114e172d1a450795d166f2b6dfd847 [ 905.387730] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d4da2b075bf24ed9be4a915b145fecdc [ 905.418527] env[64020]: DEBUG nova.network.neutron [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 905.419012] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg d4db416019c34b71ad2121939908b4b4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 905.429510] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d4db416019c34b71ad2121939908b4b4 [ 905.506086] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407918, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.062582} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 905.506365] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 905.507183] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-561df870-6c3d-4574-bf99-c0def8a8de09 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.526240] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Reconfiguring VM instance instance-0000003e to attach disk [datastore1] 7ba3e477-a789-4beb-90ac-651520836a5d/7ba3e477-a789-4beb-90ac-651520836a5d.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 905.526494] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-11a79c97-e04e-4048-a106-251b671f8ac1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.544867] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 905.544867] env[64020]: value = "task-407919" [ 905.544867] env[64020]: _type = "Task" [ 905.544867] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 905.553687] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407919, 'name': ReconfigVM_Task} progress is 6%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 905.823461] env[64020]: DEBUG nova.compute.manager [req-e6f5235f-9c47-4b1a-9fe7-ca1126ac32d0 req-e3af2d23-c4fb-47f7-8545-6049cb7ec6b6 service nova] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Received event network-changed-2541ed1d-017e-49b6-b5c7-4737c699f975 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 905.823610] env[64020]: DEBUG nova.compute.manager [req-e6f5235f-9c47-4b1a-9fe7-ca1126ac32d0 req-e3af2d23-c4fb-47f7-8545-6049cb7ec6b6 service nova] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Refreshing instance network info cache due to event network-changed-2541ed1d-017e-49b6-b5c7-4737c699f975. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 905.823742] env[64020]: DEBUG oslo_concurrency.lockutils [req-e6f5235f-9c47-4b1a-9fe7-ca1126ac32d0 req-e3af2d23-c4fb-47f7-8545-6049cb7ec6b6 service nova] Acquiring lock "refresh_cache-414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 905.846149] env[64020]: DEBUG nova.compute.utils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 905.846764] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg d4d5edeee7284df491cbca622c6312bb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 905.847627] env[64020]: DEBUG nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 905.847793] env[64020]: DEBUG nova.network.neutron [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 905.864083] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d4d5edeee7284df491cbca622c6312bb [ 905.875349] env[64020]: DEBUG nova.network.neutron [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 905.914355] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Acquiring lock "9d199d06-86cb-4d2f-894d-1ce0ef29235f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 905.914592] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Lock "9d199d06-86cb-4d2f-894d-1ce0ef29235f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 905.920437] env[64020]: DEBUG nova.policy [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '5c63abe92324464382c60661b7e98b2a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ff4944c64af74d33bca307deacb5ed1d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 905.922044] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Releasing lock "refresh_cache-7ba3e477-a789-4beb-90ac-651520836a5d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 905.922251] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Updated the network info_cache for instance {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9997}} [ 905.923472] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 905.923992] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 905.924181] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 905.924342] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 905.924486] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 905.924631] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 905.924761] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=64020) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 905.924904] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager.update_available_resource {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 905.925229] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg fb8f21165e1e4f2eb999b72790284281 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 905.934714] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb8f21165e1e4f2eb999b72790284281 [ 905.959005] env[64020]: DEBUG nova.network.neutron [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 905.959581] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg a2e5b0819011411e96293a10fcd84375 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 905.971634] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2e5b0819011411e96293a10fcd84375 [ 906.056795] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407919, 'name': ReconfigVM_Task, 'duration_secs': 0.273035} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 906.057237] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Reconfigured VM instance instance-0000003e to attach disk [datastore1] 7ba3e477-a789-4beb-90ac-651520836a5d/7ba3e477-a789-4beb-90ac-651520836a5d.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 906.058193] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-42d6a5da-9a42-41f8-8a82-85efefa1631f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.066994] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 906.066994] env[64020]: value = "task-407920" [ 906.066994] env[64020]: _type = "Task" [ 906.066994] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 906.083216] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407920, 'name': Rename_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 906.155634] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b491e45b-ea97-4f95-82ea-af0827833d9e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.167382] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca3d4b0e-070c-4b92-976c-db17cb9cac21 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.198987] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a506201b-6c00-4be3-a191-d549c600be28 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.207050] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52e6c343-877a-44a8-8f75-34312771a61c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.213206] env[64020]: DEBUG nova.network.neutron [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Successfully created port: 4728632c-dd93-41a7-8b89-dd53d70a6dd6 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 906.226825] env[64020]: DEBUG nova.compute.provider_tree [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 906.227429] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 5ae1e845279340d2a63c4bc9ebfe24b2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 906.237209] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ae1e845279340d2a63c4bc9ebfe24b2 [ 906.355683] env[64020]: DEBUG nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 906.357498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 107154952bbb4ca6ba405ab1263324eb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 906.395253] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 107154952bbb4ca6ba405ab1263324eb [ 906.428497] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 906.468204] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Releasing lock "refresh_cache-414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 906.468649] env[64020]: DEBUG nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 906.468856] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 906.469305] env[64020]: DEBUG oslo_concurrency.lockutils [req-e6f5235f-9c47-4b1a-9fe7-ca1126ac32d0 req-e3af2d23-c4fb-47f7-8545-6049cb7ec6b6 service nova] Acquired lock "refresh_cache-414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 906.469493] env[64020]: DEBUG nova.network.neutron [req-e6f5235f-9c47-4b1a-9fe7-ca1126ac32d0 req-e3af2d23-c4fb-47f7-8545-6049cb7ec6b6 service nova] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Refreshing network info cache for port 2541ed1d-017e-49b6-b5c7-4737c699f975 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 906.469920] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e6f5235f-9c47-4b1a-9fe7-ca1126ac32d0 req-e3af2d23-c4fb-47f7-8545-6049cb7ec6b6 service nova] Expecting reply to msg 6739d55b61ac4bb8a99c8772b7d46779 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 906.470718] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-94ecc38c-ff82-4cda-97f3-f957d2e6c29a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.476961] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6739d55b61ac4bb8a99c8772b7d46779 [ 906.481280] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b603bd31-5397-4e0b-84cf-cf890390fa8e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.504113] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1 could not be found. [ 906.504332] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 906.504484] env[64020]: INFO nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Took 0.04 seconds to destroy the instance on the hypervisor. [ 906.504726] env[64020]: DEBUG oslo.service.loopingcall [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 906.505140] env[64020]: DEBUG nova.compute.manager [-] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 906.505140] env[64020]: DEBUG nova.network.neutron [-] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 906.534259] env[64020]: DEBUG nova.network.neutron [-] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 906.534778] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 16ab0d640e624d5b9ace1a9cdc84be35 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 906.541681] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16ab0d640e624d5b9ace1a9cdc84be35 [ 906.576253] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407920, 'name': Rename_Task, 'duration_secs': 0.193234} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 906.576532] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 906.576769] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-52eba26e-896b-4d2d-82ac-797d39b117cd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.583209] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 906.583209] env[64020]: value = "task-407921" [ 906.583209] env[64020]: _type = "Task" [ 906.583209] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 906.591959] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407921, 'name': PowerOnVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 906.730472] env[64020]: DEBUG nova.scheduler.client.report [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 906.733004] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 7ff14ec75a804ad0a01713e7e4d1aeff in queue reply_57893177120949e6a93cb88e15cd42b4 [ 906.745484] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ff14ec75a804ad0a01713e7e4d1aeff [ 906.862484] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg e4b4aff38beb4e06aa54d136e5f67c4f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 906.895143] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e4b4aff38beb4e06aa54d136e5f67c4f [ 906.989283] env[64020]: DEBUG nova.network.neutron [req-e6f5235f-9c47-4b1a-9fe7-ca1126ac32d0 req-e3af2d23-c4fb-47f7-8545-6049cb7ec6b6 service nova] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 907.032933] env[64020]: ERROR nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4728632c-dd93-41a7-8b89-dd53d70a6dd6, please check neutron logs for more information. [ 907.032933] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 907.032933] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 907.032933] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 907.032933] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 907.032933] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 907.032933] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 907.032933] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 907.032933] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 907.032933] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 907.032933] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 907.032933] env[64020]: ERROR nova.compute.manager raise self.value [ 907.032933] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 907.032933] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 907.032933] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 907.032933] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 907.033474] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 907.033474] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 907.033474] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4728632c-dd93-41a7-8b89-dd53d70a6dd6, please check neutron logs for more information. [ 907.033474] env[64020]: ERROR nova.compute.manager [ 907.033474] env[64020]: Traceback (most recent call last): [ 907.033474] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 907.033474] env[64020]: listener.cb(fileno) [ 907.033474] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 907.033474] env[64020]: result = function(*args, **kwargs) [ 907.033474] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 907.033474] env[64020]: return func(*args, **kwargs) [ 907.033474] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 907.033474] env[64020]: raise e [ 907.033474] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 907.033474] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 907.033474] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 907.033474] env[64020]: created_port_ids = self._update_ports_for_instance( [ 907.033474] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 907.033474] env[64020]: with excutils.save_and_reraise_exception(): [ 907.033474] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 907.033474] env[64020]: self.force_reraise() [ 907.033474] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 907.033474] env[64020]: raise self.value [ 907.033474] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 907.033474] env[64020]: updated_port = self._update_port( [ 907.033474] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 907.033474] env[64020]: _ensure_no_port_binding_failure(port) [ 907.033474] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 907.033474] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 907.034379] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 4728632c-dd93-41a7-8b89-dd53d70a6dd6, please check neutron logs for more information. [ 907.034379] env[64020]: Removing descriptor: 18 [ 907.036896] env[64020]: DEBUG nova.network.neutron [-] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 907.037360] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6be41af616b44c34afea2dcab655ee93 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 907.046015] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6be41af616b44c34afea2dcab655ee93 [ 907.083358] env[64020]: DEBUG nova.network.neutron [req-e6f5235f-9c47-4b1a-9fe7-ca1126ac32d0 req-e3af2d23-c4fb-47f7-8545-6049cb7ec6b6 service nova] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 907.084105] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e6f5235f-9c47-4b1a-9fe7-ca1126ac32d0 req-e3af2d23-c4fb-47f7-8545-6049cb7ec6b6 service nova] Expecting reply to msg cf68bc9f9bb1402dbf224e53b256b6a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 907.093662] env[64020]: DEBUG oslo_vmware.api [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407921, 'name': PowerOnVM_Task, 'duration_secs': 0.427142} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 907.093916] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 907.094117] env[64020]: DEBUG nova.compute.manager [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 907.095298] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9795d74f-d139-4ada-b7a6-2a16c0625e86 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.098291] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cf68bc9f9bb1402dbf224e53b256b6a2 [ 907.102321] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 658265c401be403e993e846c8010638d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 907.133630] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 658265c401be403e993e846c8010638d [ 907.235706] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.895s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 907.236380] env[64020]: ERROR nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3c981ae7-e113-416e-a449-8d35634800be, please check neutron logs for more information. [ 907.236380] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Traceback (most recent call last): [ 907.236380] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 907.236380] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] self.driver.spawn(context, instance, image_meta, [ 907.236380] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 907.236380] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 907.236380] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 907.236380] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] vm_ref = self.build_virtual_machine(instance, [ 907.236380] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 907.236380] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] vif_infos = vmwarevif.get_vif_info(self._session, [ 907.236380] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] for vif in network_info: [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] return self._sync_wrapper(fn, *args, **kwargs) [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] self.wait() [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] self[:] = self._gt.wait() [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] return self._exit_event.wait() [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] result = hub.switch() [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 907.236694] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] return self.greenlet.switch() [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] result = function(*args, **kwargs) [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] return func(*args, **kwargs) [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] raise e [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] nwinfo = self.network_api.allocate_for_instance( [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] created_port_ids = self._update_ports_for_instance( [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] with excutils.save_and_reraise_exception(): [ 907.237007] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 907.237315] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] self.force_reraise() [ 907.237315] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 907.237315] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] raise self.value [ 907.237315] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 907.237315] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] updated_port = self._update_port( [ 907.237315] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 907.237315] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] _ensure_no_port_binding_failure(port) [ 907.237315] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 907.237315] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] raise exception.PortBindingFailed(port_id=port['id']) [ 907.237315] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] nova.exception.PortBindingFailed: Binding failed for port 3c981ae7-e113-416e-a449-8d35634800be, please check neutron logs for more information. [ 907.237315] env[64020]: ERROR nova.compute.manager [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] [ 907.237634] env[64020]: DEBUG nova.compute.utils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Binding failed for port 3c981ae7-e113-416e-a449-8d35634800be, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 907.238343] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.323s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 907.239769] env[64020]: INFO nova.compute.claims [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 907.241348] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 2943238c81d2469ead366b42800fa48c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 907.244107] env[64020]: DEBUG nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Build of instance 8c4773b7-a8de-4de3-a91f-2252cc34d11b was re-scheduled: Binding failed for port 3c981ae7-e113-416e-a449-8d35634800be, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 907.244497] env[64020]: DEBUG nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 907.244715] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Acquiring lock "refresh_cache-8c4773b7-a8de-4de3-a91f-2252cc34d11b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 907.244858] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Acquired lock "refresh_cache-8c4773b7-a8de-4de3-a91f-2252cc34d11b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 907.245012] env[64020]: DEBUG nova.network.neutron [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 907.245387] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg d60b3805a26046f59f07f26fe31ad901 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 907.251112] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d60b3805a26046f59f07f26fe31ad901 [ 907.275692] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2943238c81d2469ead366b42800fa48c [ 907.367348] env[64020]: DEBUG nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 907.391909] env[64020]: DEBUG nova.virt.hardware [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 907.392151] env[64020]: DEBUG nova.virt.hardware [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 907.392309] env[64020]: DEBUG nova.virt.hardware [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 907.392486] env[64020]: DEBUG nova.virt.hardware [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 907.392627] env[64020]: DEBUG nova.virt.hardware [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 907.392766] env[64020]: DEBUG nova.virt.hardware [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 907.392963] env[64020]: DEBUG nova.virt.hardware [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 907.393114] env[64020]: DEBUG nova.virt.hardware [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 907.393272] env[64020]: DEBUG nova.virt.hardware [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 907.393428] env[64020]: DEBUG nova.virt.hardware [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 907.393628] env[64020]: DEBUG nova.virt.hardware [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 907.394439] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55532e30-059f-41e2-b32c-e76d44a50801 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.401960] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ea5d27d-19be-4aaa-bb24-cabbc0ea9579 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.415482] env[64020]: ERROR nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4728632c-dd93-41a7-8b89-dd53d70a6dd6, please check neutron logs for more information. [ 907.415482] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Traceback (most recent call last): [ 907.415482] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 907.415482] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] yield resources [ 907.415482] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 907.415482] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] self.driver.spawn(context, instance, image_meta, [ 907.415482] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 907.415482] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 907.415482] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 907.415482] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] vm_ref = self.build_virtual_machine(instance, [ 907.415482] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] vif_infos = vmwarevif.get_vif_info(self._session, [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] for vif in network_info: [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] return self._sync_wrapper(fn, *args, **kwargs) [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] self.wait() [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] self[:] = self._gt.wait() [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] return self._exit_event.wait() [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 907.415906] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] current.throw(*self._exc) [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] result = function(*args, **kwargs) [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] return func(*args, **kwargs) [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] raise e [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] nwinfo = self.network_api.allocate_for_instance( [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] created_port_ids = self._update_ports_for_instance( [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] with excutils.save_and_reraise_exception(): [ 907.416297] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 907.416673] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] self.force_reraise() [ 907.416673] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 907.416673] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] raise self.value [ 907.416673] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 907.416673] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] updated_port = self._update_port( [ 907.416673] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 907.416673] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] _ensure_no_port_binding_failure(port) [ 907.416673] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 907.416673] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] raise exception.PortBindingFailed(port_id=port['id']) [ 907.416673] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] nova.exception.PortBindingFailed: Binding failed for port 4728632c-dd93-41a7-8b89-dd53d70a6dd6, please check neutron logs for more information. [ 907.416673] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] [ 907.416673] env[64020]: INFO nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Terminating instance [ 907.417715] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Acquiring lock "refresh_cache-a5567127-3d68-4c37-a8aa-03c5579cc18c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 907.417869] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Acquired lock "refresh_cache-a5567127-3d68-4c37-a8aa-03c5579cc18c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 907.418029] env[64020]: DEBUG nova.network.neutron [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 907.418434] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 54ea8274d063402ebbf189934b0c254c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 907.425123] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54ea8274d063402ebbf189934b0c254c [ 907.492949] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 917013f42f834552bc6b9e5762a1653e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 907.501458] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 917013f42f834552bc6b9e5762a1653e [ 907.539061] env[64020]: INFO nova.compute.manager [-] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Took 1.03 seconds to deallocate network for instance. [ 907.541421] env[64020]: DEBUG nova.compute.claims [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 907.541596] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 907.589435] env[64020]: DEBUG oslo_concurrency.lockutils [req-e6f5235f-9c47-4b1a-9fe7-ca1126ac32d0 req-e3af2d23-c4fb-47f7-8545-6049cb7ec6b6 service nova] Releasing lock "refresh_cache-414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 907.589745] env[64020]: DEBUG nova.compute.manager [req-e6f5235f-9c47-4b1a-9fe7-ca1126ac32d0 req-e3af2d23-c4fb-47f7-8545-6049cb7ec6b6 service nova] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Received event network-vif-deleted-2541ed1d-017e-49b6-b5c7-4737c699f975 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 907.612426] env[64020]: DEBUG oslo_concurrency.lockutils [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 907.748812] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 4e188bceae9e4265b8bf3b245cb6f666 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 907.756665] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e188bceae9e4265b8bf3b245cb6f666 [ 907.789593] env[64020]: DEBUG nova.network.neutron [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 907.873624] env[64020]: DEBUG nova.network.neutron [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 907.874219] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 405bd9b965344e02a9ed1d6bd85661c5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 907.881960] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 405bd9b965344e02a9ed1d6bd85661c5 [ 907.925942] env[64020]: DEBUG nova.compute.manager [req-fc0ae337-2244-48c4-8064-2690049a5cb8 req-4aff8224-15d8-44f1-aac1-0bc70d1f435e service nova] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Received event network-changed-4728632c-dd93-41a7-8b89-dd53d70a6dd6 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 907.926135] env[64020]: DEBUG nova.compute.manager [req-fc0ae337-2244-48c4-8064-2690049a5cb8 req-4aff8224-15d8-44f1-aac1-0bc70d1f435e service nova] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Refreshing instance network info cache due to event network-changed-4728632c-dd93-41a7-8b89-dd53d70a6dd6. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 907.926322] env[64020]: DEBUG oslo_concurrency.lockutils [req-fc0ae337-2244-48c4-8064-2690049a5cb8 req-4aff8224-15d8-44f1-aac1-0bc70d1f435e service nova] Acquiring lock "refresh_cache-a5567127-3d68-4c37-a8aa-03c5579cc18c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 907.964271] env[64020]: DEBUG nova.network.neutron [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 908.070572] env[64020]: DEBUG nova.network.neutron [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 908.071097] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 009597202a21481a8d1ebdce8cd96bfa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 908.079389] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 009597202a21481a8d1ebdce8cd96bfa [ 908.231689] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg a896d758100e4d90823848b8a817fbb8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 908.241676] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a896d758100e4d90823848b8a817fbb8 [ 908.376765] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Releasing lock "refresh_cache-8c4773b7-a8de-4de3-a91f-2252cc34d11b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 908.377051] env[64020]: DEBUG nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 908.377171] env[64020]: DEBUG nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 908.377331] env[64020]: DEBUG nova.network.neutron [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 908.406451] env[64020]: DEBUG nova.network.neutron [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 908.407099] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 7817782ed270454b9eeac6d87e15bc46 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 908.417035] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7817782ed270454b9eeac6d87e15bc46 [ 908.573840] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Releasing lock "refresh_cache-a5567127-3d68-4c37-a8aa-03c5579cc18c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 908.574254] env[64020]: DEBUG nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 908.574449] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 908.575506] env[64020]: DEBUG oslo_concurrency.lockutils [req-fc0ae337-2244-48c4-8064-2690049a5cb8 req-4aff8224-15d8-44f1-aac1-0bc70d1f435e service nova] Acquired lock "refresh_cache-a5567127-3d68-4c37-a8aa-03c5579cc18c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 908.575688] env[64020]: DEBUG nova.network.neutron [req-fc0ae337-2244-48c4-8064-2690049a5cb8 req-4aff8224-15d8-44f1-aac1-0bc70d1f435e service nova] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Refreshing network info cache for port 4728632c-dd93-41a7-8b89-dd53d70a6dd6 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 908.576115] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-fc0ae337-2244-48c4-8064-2690049a5cb8 req-4aff8224-15d8-44f1-aac1-0bc70d1f435e service nova] Expecting reply to msg 0805632c66034b12aa8506f367729232 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 908.576988] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-6697f252-cb84-42d8-9ea2-ab043e1e36f7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.579655] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9cd3a11f-c0be-4464-aa7d-43beab10fb52 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.582930] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0805632c66034b12aa8506f367729232 [ 908.589554] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffab8222-2310-43c5-bdd2-4745edc5889d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.595028] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9043fc6c-b371-4764-8aa5-c2c57dd56756 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.646276] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e49ac3c-3ced-48c0-9d6f-35d084965354 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.649200] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a5567127-3d68-4c37-a8aa-03c5579cc18c could not be found. [ 908.649435] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 908.649942] env[64020]: INFO nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Took 0.08 seconds to destroy the instance on the hypervisor. [ 908.650293] env[64020]: DEBUG oslo.service.loopingcall [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 908.650580] env[64020]: DEBUG nova.compute.manager [-] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 908.650671] env[64020]: DEBUG nova.network.neutron [-] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 908.658944] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-edbe3fb6-dd41-4fd6-851a-d85c29ec6335 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.663998] env[64020]: DEBUG nova.network.neutron [-] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 908.664481] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6c8988b97cbd4dbc8175be7197844c12 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 908.672649] env[64020]: DEBUG nova.compute.provider_tree [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 908.673098] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg fef9d64c8ee84ee3b4e4c40a4785f8be in queue reply_57893177120949e6a93cb88e15cd42b4 [ 908.674377] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6c8988b97cbd4dbc8175be7197844c12 [ 908.679373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fef9d64c8ee84ee3b4e4c40a4785f8be [ 908.733571] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquiring lock "7ba3e477-a789-4beb-90ac-651520836a5d" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 908.733805] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "7ba3e477-a789-4beb-90ac-651520836a5d" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 908.734095] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquiring lock "7ba3e477-a789-4beb-90ac-651520836a5d-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 908.734283] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "7ba3e477-a789-4beb-90ac-651520836a5d-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 908.734454] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "7ba3e477-a789-4beb-90ac-651520836a5d-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 908.736583] env[64020]: INFO nova.compute.manager [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Terminating instance [ 908.738181] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquiring lock "refresh_cache-7ba3e477-a789-4beb-90ac-651520836a5d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 908.738336] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquired lock "refresh_cache-7ba3e477-a789-4beb-90ac-651520836a5d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 908.738492] env[64020]: DEBUG nova.network.neutron [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 908.738882] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 7f1f253b5b404bdca5fcc0ea85b142bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 908.746329] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f1f253b5b404bdca5fcc0ea85b142bc [ 908.911318] env[64020]: DEBUG nova.network.neutron [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 908.911875] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg be9167c6a2d34a39a907f0b0f9b702a6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 908.919644] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be9167c6a2d34a39a907f0b0f9b702a6 [ 909.099018] env[64020]: DEBUG nova.network.neutron [req-fc0ae337-2244-48c4-8064-2690049a5cb8 req-4aff8224-15d8-44f1-aac1-0bc70d1f435e service nova] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 909.174538] env[64020]: DEBUG nova.network.neutron [-] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 909.175048] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg fd90ff4cf1844aee8530614162c24677 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 909.183779] env[64020]: DEBUG nova.scheduler.client.report [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 909.186468] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg d1ca91ce85594d409b4a72b632946daa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 909.187620] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd90ff4cf1844aee8530614162c24677 [ 909.188731] env[64020]: DEBUG nova.network.neutron [req-fc0ae337-2244-48c4-8064-2690049a5cb8 req-4aff8224-15d8-44f1-aac1-0bc70d1f435e service nova] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 909.189170] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-fc0ae337-2244-48c4-8064-2690049a5cb8 req-4aff8224-15d8-44f1-aac1-0bc70d1f435e service nova] Expecting reply to msg 72aa59a23ced4f55ab8130b75961f560 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 909.190086] env[64020]: INFO nova.compute.manager [-] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Took 0.54 seconds to deallocate network for instance. [ 909.192408] env[64020]: DEBUG nova.compute.claims [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 909.192588] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 909.199876] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1ca91ce85594d409b4a72b632946daa [ 909.200342] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72aa59a23ced4f55ab8130b75961f560 [ 909.254615] env[64020]: DEBUG nova.network.neutron [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 909.300157] env[64020]: DEBUG nova.network.neutron [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 909.300752] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 0902b27e9cb248d88f470e5b6b04a453 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 909.309119] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0902b27e9cb248d88f470e5b6b04a453 [ 909.413910] env[64020]: INFO nova.compute.manager [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] [instance: 8c4773b7-a8de-4de3-a91f-2252cc34d11b] Took 1.04 seconds to deallocate network for instance. [ 909.416195] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg b99d9aba0a954664adafb44ee32b0faa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 909.451952] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b99d9aba0a954664adafb44ee32b0faa [ 909.693581] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.455s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 909.694135] env[64020]: DEBUG nova.compute.manager [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 909.695764] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 6482fe96ea4a4e2b93b5c2da7deb6260 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 909.697248] env[64020]: DEBUG oslo_concurrency.lockutils [req-fc0ae337-2244-48c4-8064-2690049a5cb8 req-4aff8224-15d8-44f1-aac1-0bc70d1f435e service nova] Releasing lock "refresh_cache-a5567127-3d68-4c37-a8aa-03c5579cc18c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 909.697457] env[64020]: DEBUG nova.compute.manager [req-fc0ae337-2244-48c4-8064-2690049a5cb8 req-4aff8224-15d8-44f1-aac1-0bc70d1f435e service nova] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Received event network-vif-deleted-4728632c-dd93-41a7-8b89-dd53d70a6dd6 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 909.698025] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.959s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 909.699411] env[64020]: INFO nova.compute.claims [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 909.700968] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 59ad6f9c04874c9d9a4022b4868d5e91 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 909.733869] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59ad6f9c04874c9d9a4022b4868d5e91 [ 909.735173] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6482fe96ea4a4e2b93b5c2da7deb6260 [ 909.803134] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Releasing lock "refresh_cache-7ba3e477-a789-4beb-90ac-651520836a5d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 909.803535] env[64020]: DEBUG nova.compute.manager [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 909.803724] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 909.804657] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a26d2725-99f7-4b27-bca3-d2bb2d8c4ff7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.812795] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 909.813041] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-f2e047d8-760c-4bbb-a040-914e1d25cfdb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.818885] env[64020]: DEBUG oslo_vmware.api [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 909.818885] env[64020]: value = "task-407922" [ 909.818885] env[64020]: _type = "Task" [ 909.818885] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 909.827075] env[64020]: DEBUG oslo_vmware.api [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407922, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 909.932512] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 7f2f007b34104e32aff59cd2c871a7ab in queue reply_57893177120949e6a93cb88e15cd42b4 [ 909.965426] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f2f007b34104e32aff59cd2c871a7ab [ 910.199896] env[64020]: DEBUG nova.compute.utils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 910.200567] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg dfb2efabcb8d4e78a42f70f35c477d59 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 910.201941] env[64020]: DEBUG nova.compute.manager [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Not allocating networking since 'none' was specified. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 910.203441] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg ba7232063b6a4119b228774294ac9679 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 910.210429] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dfb2efabcb8d4e78a42f70f35c477d59 [ 910.214123] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba7232063b6a4119b228774294ac9679 [ 910.328896] env[64020]: DEBUG oslo_vmware.api [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407922, 'name': PowerOffVM_Task, 'duration_secs': 0.178408} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 910.329127] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 910.329291] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 910.329524] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-54d64c98-3531-4290-9fce-6c208af9992b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.354917] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 910.355113] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Deleting contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 910.355289] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Deleting the datastore file [datastore1] 7ba3e477-a789-4beb-90ac-651520836a5d {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 910.355514] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-6c663108-f82c-4c79-b761-29e0b14b16bf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.361728] env[64020]: DEBUG oslo_vmware.api [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for the task: (returnval){ [ 910.361728] env[64020]: value = "task-407924" [ 910.361728] env[64020]: _type = "Task" [ 910.361728] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 910.368738] env[64020]: DEBUG oslo_vmware.api [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407924, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 910.453727] env[64020]: INFO nova.scheduler.client.report [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Deleted allocations for instance 8c4773b7-a8de-4de3-a91f-2252cc34d11b [ 910.460656] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Expecting reply to msg 42ddb06c637c4b6d8b7d42700f0cd69b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 910.471257] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42ddb06c637c4b6d8b7d42700f0cd69b [ 910.703631] env[64020]: DEBUG nova.compute.manager [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 910.705599] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg d5bf7be7540f4cdab9e036d22c67f198 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 910.737847] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5bf7be7540f4cdab9e036d22c67f198 [ 910.873716] env[64020]: DEBUG oslo_vmware.api [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Task: {'id': task-407924, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.096553} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 910.873983] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 910.874161] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Deleted contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 910.874337] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 910.874504] env[64020]: INFO nova.compute.manager [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Took 1.07 seconds to destroy the instance on the hypervisor. [ 910.874754] env[64020]: DEBUG oslo.service.loopingcall [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 910.874925] env[64020]: DEBUG nova.compute.manager [-] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 910.875016] env[64020]: DEBUG nova.network.neutron [-] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 910.908343] env[64020]: DEBUG nova.network.neutron [-] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 910.908842] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg fe8bf3926a1e4405a918e61cf95d63d0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 910.916897] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe8bf3926a1e4405a918e61cf95d63d0 [ 910.948060] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2e4290e-1419-4ea8-8509-6d92a4fa97e7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.956158] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35da64e6-b991-4e7e-9078-d70e20c5a94d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.962338] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4edbbe7e-daf8-4fb5-854e-bf8108887323 tempest-ServerMetadataNegativeTestJSON-1604332549 tempest-ServerMetadataNegativeTestJSON-1604332549-project-member] Lock "8c4773b7-a8de-4de3-a91f-2252cc34d11b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 130.883s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 910.989347] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 9d70ea182d1144649027b9ab4a6d4d25 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 910.991456] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84b1414b-b72a-47f9-bb79-96187530bfd4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.000028] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac06e347-65bc-4058-b242-5b1253dd1c38 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.003864] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9d70ea182d1144649027b9ab4a6d4d25 [ 911.016300] env[64020]: DEBUG nova.compute.provider_tree [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 911.017148] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 6706650bbf4644869b0a498641aea6e7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 911.025401] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6706650bbf4644869b0a498641aea6e7 [ 911.210575] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 42230a0059574064acac2b7b681b5d83 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 911.241562] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42230a0059574064acac2b7b681b5d83 [ 911.413090] env[64020]: DEBUG nova.network.neutron [-] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 911.413654] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 42f2d8d2745344ee9e1bb986a98d57bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 911.421948] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42f2d8d2745344ee9e1bb986a98d57bc [ 911.491880] env[64020]: DEBUG nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 911.493669] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 1d4ae4257d0d4fa9b16d06fc23932006 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 911.519891] env[64020]: DEBUG nova.scheduler.client.report [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 911.522375] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 7b8a0a58685e40e1b03eb3324ee39d32 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 911.529073] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d4ae4257d0d4fa9b16d06fc23932006 [ 911.535776] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7b8a0a58685e40e1b03eb3324ee39d32 [ 911.714101] env[64020]: DEBUG nova.compute.manager [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 911.738228] env[64020]: DEBUG nova.virt.hardware [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 911.738529] env[64020]: DEBUG nova.virt.hardware [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 911.738725] env[64020]: DEBUG nova.virt.hardware [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 911.738946] env[64020]: DEBUG nova.virt.hardware [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 911.739129] env[64020]: DEBUG nova.virt.hardware [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 911.739338] env[64020]: DEBUG nova.virt.hardware [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 911.739592] env[64020]: DEBUG nova.virt.hardware [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 911.739806] env[64020]: DEBUG nova.virt.hardware [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 911.740030] env[64020]: DEBUG nova.virt.hardware [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 911.740226] env[64020]: DEBUG nova.virt.hardware [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 911.740434] env[64020]: DEBUG nova.virt.hardware [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 911.741344] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90eeddb9-e5f1-4a1a-b94a-daf1e3ec4ccb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.749916] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f706fbbe-79f4-40aa-b036-f828c47eff95 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.763624] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 911.769270] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Creating folder: Project (ad588f6a8c8d491eb80df9e73cfc985e). Parent ref: group-v110249. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 911.769560] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-503eccc8-83ca-4bbf-bab2-bb543d625da7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.778881] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Created folder: Project (ad588f6a8c8d491eb80df9e73cfc985e) in parent group-v110249. [ 911.779099] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Creating folder: Instances. Parent ref: group-v110274. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 911.779411] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-26089071-00c1-41eb-9eda-fc5f8392e90a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.787720] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Created folder: Instances in parent group-v110274. [ 911.787945] env[64020]: DEBUG oslo.service.loopingcall [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 911.788190] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 911.788385] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-d9638af3-2c9a-4e60-b9f7-08a2c63fb8ef {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.805874] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 911.805874] env[64020]: value = "task-407927" [ 911.805874] env[64020]: _type = "Task" [ 911.805874] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 911.812782] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407927, 'name': CreateVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 911.915513] env[64020]: INFO nova.compute.manager [-] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Took 1.04 seconds to deallocate network for instance. [ 911.919614] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg cfcb0436ac744808aa4c962d3b80da21 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 911.957895] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfcb0436ac744808aa4c962d3b80da21 [ 912.021486] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 912.025082] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.327s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 912.025572] env[64020]: DEBUG nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 912.027629] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 835adefa06734284a63846728de2c548 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 912.028833] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.988s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 912.030628] env[64020]: INFO nova.compute.claims [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 912.032568] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 5caae212e5be4718b5e631f5f792eb05 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 912.075300] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 835adefa06734284a63846728de2c548 [ 912.101573] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5caae212e5be4718b5e631f5f792eb05 [ 912.316323] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407927, 'name': CreateVM_Task, 'duration_secs': 0.231749} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 912.316629] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 912.317119] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 912.317365] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 912.319172] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 912.319172] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f5d8871d-2549-447d-a840-4b249e117653 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.324954] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for the task: (returnval){ [ 912.324954] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5291b941-f30e-fd07-4672-1ffb73ff7fe3" [ 912.324954] env[64020]: _type = "Task" [ 912.324954] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 912.334409] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5291b941-f30e-fd07-4672-1ffb73ff7fe3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 912.423750] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 912.536430] env[64020]: DEBUG nova.compute.utils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 912.537087] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 97769efe338c490b84aa17e38fbf36f7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 912.539233] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg a88ecfcf2b8a4452849d2a804e74c65b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 912.540182] env[64020]: DEBUG nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 912.540352] env[64020]: DEBUG nova.network.neutron [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 912.547853] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a88ecfcf2b8a4452849d2a804e74c65b [ 912.548583] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 97769efe338c490b84aa17e38fbf36f7 [ 912.594066] env[64020]: DEBUG nova.policy [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'bc36618a56144356ac85690627457a2b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd2e3cd82953341479a772c43fa7c56ce', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 912.835968] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5291b941-f30e-fd07-4672-1ffb73ff7fe3, 'name': SearchDatastore_Task, 'duration_secs': 0.009364} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 912.836365] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 912.836645] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 912.836935] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 912.837131] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 912.837375] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 912.837671] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-115be945-e5ca-4e87-9b3e-6b851e1c2647 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.845788] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 912.846029] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 912.846762] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-109ed3df-d50d-4b5e-8887-4aa2620f74e1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.851692] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for the task: (returnval){ [ 912.851692] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52e86ad3-1bc7-cc0a-adca-8921770332b7" [ 912.851692] env[64020]: _type = "Task" [ 912.851692] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 912.858847] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52e86ad3-1bc7-cc0a-adca-8921770332b7, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 912.866645] env[64020]: DEBUG nova.network.neutron [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Successfully created port: ca85ae3e-6436-4834-b879-e023f3211066 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 913.041560] env[64020]: DEBUG nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 913.043368] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 99b8ca6c9c904b2a8f2882e595ee0eb4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 913.097005] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99b8ca6c9c904b2a8f2882e595ee0eb4 [ 913.317402] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-955befe2-2efd-4a5a-bdb6-bca30356c098 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.325064] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfb01cf2-205d-4c1a-920a-b32edc1f5497 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.365053] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-532b49c8-25f8-458f-b562-9bdebb69a5b2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.372457] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52e86ad3-1bc7-cc0a-adca-8921770332b7, 'name': SearchDatastore_Task, 'duration_secs': 0.007648} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 913.374769] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-693a2079-3abe-41fd-8486-bdcc3f8150e8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.377736] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f73f5e9-3a01-41bd-8d91-51eb7a99a2a2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.384178] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for the task: (returnval){ [ 913.384178] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52b14901-ee33-c7f1-a6f5-34559ac26f9e" [ 913.384178] env[64020]: _type = "Task" [ 913.384178] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 913.391665] env[64020]: DEBUG nova.compute.provider_tree [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 913.392202] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 1b47edc649b64396ba3751b03d1dfe64 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 913.401758] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52b14901-ee33-c7f1-a6f5-34559ac26f9e, 'name': SearchDatastore_Task, 'duration_secs': 0.008413} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 913.402272] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b47edc649b64396ba3751b03d1dfe64 [ 913.402679] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 913.402925] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 3c5781a8-df02-4b4f-ad3f-c04db921748f/3c5781a8-df02-4b4f-ad3f-c04db921748f.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 913.403163] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-f25ab8a2-1b0f-4619-8067-27c76cae885e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.409946] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for the task: (returnval){ [ 913.409946] env[64020]: value = "task-407928" [ 913.409946] env[64020]: _type = "Task" [ 913.409946] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 913.417810] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407928, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 913.550793] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 932728cc817b400ba016ebc637d307d2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 913.587903] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 932728cc817b400ba016ebc637d307d2 [ 913.700072] env[64020]: DEBUG nova.compute.manager [req-a5ec2e94-2afe-4e25-a18a-bcf4c4ab3c8b req-71449d57-6956-498d-a91e-916cff4f5713 service nova] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Received event network-changed-ca85ae3e-6436-4834-b879-e023f3211066 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 913.700288] env[64020]: DEBUG nova.compute.manager [req-a5ec2e94-2afe-4e25-a18a-bcf4c4ab3c8b req-71449d57-6956-498d-a91e-916cff4f5713 service nova] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Refreshing instance network info cache due to event network-changed-ca85ae3e-6436-4834-b879-e023f3211066. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 913.700556] env[64020]: DEBUG oslo_concurrency.lockutils [req-a5ec2e94-2afe-4e25-a18a-bcf4c4ab3c8b req-71449d57-6956-498d-a91e-916cff4f5713 service nova] Acquiring lock "refresh_cache-b1ec9141-54d5-4761-bd37-55a1fc998589" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 913.700657] env[64020]: DEBUG oslo_concurrency.lockutils [req-a5ec2e94-2afe-4e25-a18a-bcf4c4ab3c8b req-71449d57-6956-498d-a91e-916cff4f5713 service nova] Acquired lock "refresh_cache-b1ec9141-54d5-4761-bd37-55a1fc998589" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 913.700855] env[64020]: DEBUG nova.network.neutron [req-a5ec2e94-2afe-4e25-a18a-bcf4c4ab3c8b req-71449d57-6956-498d-a91e-916cff4f5713 service nova] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Refreshing network info cache for port ca85ae3e-6436-4834-b879-e023f3211066 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 913.701292] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-a5ec2e94-2afe-4e25-a18a-bcf4c4ab3c8b req-71449d57-6956-498d-a91e-916cff4f5713 service nova] Expecting reply to msg 3c775806b1f34e4fa76dfc49b71a9903 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 913.709003] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c775806b1f34e4fa76dfc49b71a9903 [ 913.895906] env[64020]: DEBUG nova.scheduler.client.report [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 913.898623] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 9273097c96f143c4aac7d074e57cbb15 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 913.902495] env[64020]: ERROR nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ca85ae3e-6436-4834-b879-e023f3211066, please check neutron logs for more information. [ 913.902495] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 913.902495] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 913.902495] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 913.902495] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 913.902495] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 913.902495] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 913.902495] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 913.902495] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 913.902495] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 913.902495] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 913.902495] env[64020]: ERROR nova.compute.manager raise self.value [ 913.902495] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 913.902495] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 913.902495] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 913.902495] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 913.902977] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 913.902977] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 913.902977] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ca85ae3e-6436-4834-b879-e023f3211066, please check neutron logs for more information. [ 913.902977] env[64020]: ERROR nova.compute.manager [ 913.902977] env[64020]: Traceback (most recent call last): [ 913.902977] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 913.902977] env[64020]: listener.cb(fileno) [ 913.902977] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 913.902977] env[64020]: result = function(*args, **kwargs) [ 913.902977] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 913.902977] env[64020]: return func(*args, **kwargs) [ 913.902977] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 913.902977] env[64020]: raise e [ 913.902977] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 913.902977] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 913.902977] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 913.902977] env[64020]: created_port_ids = self._update_ports_for_instance( [ 913.902977] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 913.902977] env[64020]: with excutils.save_and_reraise_exception(): [ 913.902977] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 913.902977] env[64020]: self.force_reraise() [ 913.902977] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 913.902977] env[64020]: raise self.value [ 913.902977] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 913.902977] env[64020]: updated_port = self._update_port( [ 913.902977] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 913.902977] env[64020]: _ensure_no_port_binding_failure(port) [ 913.902977] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 913.902977] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 913.903775] env[64020]: nova.exception.PortBindingFailed: Binding failed for port ca85ae3e-6436-4834-b879-e023f3211066, please check neutron logs for more information. [ 913.903775] env[64020]: Removing descriptor: 16 [ 913.909587] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9273097c96f143c4aac7d074e57cbb15 [ 913.921236] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407928, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.480368} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 913.921499] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 3c5781a8-df02-4b4f-ad3f-c04db921748f/3c5781a8-df02-4b4f-ad3f-c04db921748f.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 913.921706] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 913.921951] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-33a7572a-0d4c-4f4b-9b02-438be1eb2acc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.928705] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for the task: (returnval){ [ 913.928705] env[64020]: value = "task-407929" [ 913.928705] env[64020]: _type = "Task" [ 913.928705] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 913.937148] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407929, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 914.065502] env[64020]: DEBUG nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 914.091462] env[64020]: DEBUG nova.virt.hardware [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 914.091732] env[64020]: DEBUG nova.virt.hardware [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 914.091887] env[64020]: DEBUG nova.virt.hardware [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 914.092081] env[64020]: DEBUG nova.virt.hardware [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 914.092282] env[64020]: DEBUG nova.virt.hardware [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 914.092427] env[64020]: DEBUG nova.virt.hardware [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 914.092634] env[64020]: DEBUG nova.virt.hardware [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 914.092789] env[64020]: DEBUG nova.virt.hardware [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 914.092952] env[64020]: DEBUG nova.virt.hardware [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 914.093108] env[64020]: DEBUG nova.virt.hardware [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 914.093306] env[64020]: DEBUG nova.virt.hardware [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 914.094188] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f74a6556-2fed-4304-8a11-b6ccf211b07b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.101866] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0abb43c1-e25a-478a-a0a1-d84575a1fec7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.115326] env[64020]: ERROR nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ca85ae3e-6436-4834-b879-e023f3211066, please check neutron logs for more information. [ 914.115326] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Traceback (most recent call last): [ 914.115326] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 914.115326] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] yield resources [ 914.115326] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 914.115326] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] self.driver.spawn(context, instance, image_meta, [ 914.115326] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 914.115326] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] self._vmops.spawn(context, instance, image_meta, injected_files, [ 914.115326] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 914.115326] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] vm_ref = self.build_virtual_machine(instance, [ 914.115326] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] vif_infos = vmwarevif.get_vif_info(self._session, [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] for vif in network_info: [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] return self._sync_wrapper(fn, *args, **kwargs) [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] self.wait() [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] self[:] = self._gt.wait() [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] return self._exit_event.wait() [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 914.116103] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] current.throw(*self._exc) [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] result = function(*args, **kwargs) [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] return func(*args, **kwargs) [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] raise e [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] nwinfo = self.network_api.allocate_for_instance( [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] created_port_ids = self._update_ports_for_instance( [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] with excutils.save_and_reraise_exception(): [ 914.116893] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 914.117709] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] self.force_reraise() [ 914.117709] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 914.117709] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] raise self.value [ 914.117709] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 914.117709] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] updated_port = self._update_port( [ 914.117709] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 914.117709] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] _ensure_no_port_binding_failure(port) [ 914.117709] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 914.117709] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] raise exception.PortBindingFailed(port_id=port['id']) [ 914.117709] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] nova.exception.PortBindingFailed: Binding failed for port ca85ae3e-6436-4834-b879-e023f3211066, please check neutron logs for more information. [ 914.117709] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] [ 914.117709] env[64020]: INFO nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Terminating instance [ 914.118453] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquiring lock "refresh_cache-b1ec9141-54d5-4761-bd37-55a1fc998589" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 914.220764] env[64020]: DEBUG nova.network.neutron [req-a5ec2e94-2afe-4e25-a18a-bcf4c4ab3c8b req-71449d57-6956-498d-a91e-916cff4f5713 service nova] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 914.366822] env[64020]: DEBUG nova.network.neutron [req-a5ec2e94-2afe-4e25-a18a-bcf4c4ab3c8b req-71449d57-6956-498d-a91e-916cff4f5713 service nova] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 914.366822] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-a5ec2e94-2afe-4e25-a18a-bcf4c4ab3c8b req-71449d57-6956-498d-a91e-916cff4f5713 service nova] Expecting reply to msg a18a9bae99b7410a93ac6ab31c09dbcd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 914.374842] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a18a9bae99b7410a93ac6ab31c09dbcd [ 914.404485] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.372s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 914.404485] env[64020]: DEBUG nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 914.404485] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg ff040f4fc01b4dd1a42d1b05e1caeef2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 914.405486] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.575s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 914.408257] env[64020]: INFO nova.compute.claims [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 914.409894] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 9b36b298589f433ab5ca51d0f4021d63 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 914.439524] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407929, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.06547} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 914.441030] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff040f4fc01b4dd1a42d1b05e1caeef2 [ 914.441753] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 914.442679] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a035af4-4b78-416a-b87c-909ea229d96d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.462460] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b36b298589f433ab5ca51d0f4021d63 [ 914.471264] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Reconfiguring VM instance instance-00000044 to attach disk [datastore1] 3c5781a8-df02-4b4f-ad3f-c04db921748f/3c5781a8-df02-4b4f-ad3f-c04db921748f.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 914.471907] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-2d4b7a1a-6bcd-4843-8e69-9f4e7ca636bf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.492936] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for the task: (returnval){ [ 914.492936] env[64020]: value = "task-407930" [ 914.492936] env[64020]: _type = "Task" [ 914.492936] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 914.500972] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407930, 'name': ReconfigVM_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 914.870315] env[64020]: DEBUG oslo_concurrency.lockutils [req-a5ec2e94-2afe-4e25-a18a-bcf4c4ab3c8b req-71449d57-6956-498d-a91e-916cff4f5713 service nova] Releasing lock "refresh_cache-b1ec9141-54d5-4761-bd37-55a1fc998589" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 914.870817] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquired lock "refresh_cache-b1ec9141-54d5-4761-bd37-55a1fc998589" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 914.871040] env[64020]: DEBUG nova.network.neutron [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 914.871691] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 16c5060eb7e04b749f2246135933245f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 914.883868] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16c5060eb7e04b749f2246135933245f [ 914.919827] env[64020]: DEBUG nova.compute.utils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 914.919827] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg c2ce7255b61b49108946f07a43d9752a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 914.944829] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg b8980e34b4cb465986694480651955e4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 914.944829] env[64020]: DEBUG nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 914.944829] env[64020]: DEBUG nova.network.neutron [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 914.944829] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8980e34b4cb465986694480651955e4 [ 914.944829] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2ce7255b61b49108946f07a43d9752a [ 914.971836] env[64020]: DEBUG nova.policy [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'bc36618a56144356ac85690627457a2b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd2e3cd82953341479a772c43fa7c56ce', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 915.005647] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407930, 'name': ReconfigVM_Task, 'duration_secs': 0.303757} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 915.006134] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Reconfigured VM instance instance-00000044 to attach disk [datastore1] 3c5781a8-df02-4b4f-ad3f-c04db921748f/3c5781a8-df02-4b4f-ad3f-c04db921748f.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 915.006760] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-77a6f152-75f7-450b-84f8-df0705d8abfa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.012899] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for the task: (returnval){ [ 915.012899] env[64020]: value = "task-407931" [ 915.012899] env[64020]: _type = "Task" [ 915.012899] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 915.020611] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407931, 'name': Rename_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 915.275733] env[64020]: DEBUG nova.network.neutron [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Successfully created port: 113dd114-5149-4e96-9b3d-599793e830c8 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 915.393013] env[64020]: DEBUG nova.network.neutron [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 915.424868] env[64020]: DEBUG nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 915.426562] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 9fff004a41f24a9387ee4d1bb56f4e6a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 915.474481] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9fff004a41f24a9387ee4d1bb56f4e6a [ 915.480673] env[64020]: DEBUG nova.network.neutron [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 915.481270] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg cd586a9ec00b48e7bf0e1c0399e872d5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 915.488953] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd586a9ec00b48e7bf0e1c0399e872d5 [ 915.527440] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407931, 'name': Rename_Task, 'duration_secs': 0.142721} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 915.528172] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 915.528455] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-1f77c28e-3896-490a-bfce-fc2db081945c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.543436] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for the task: (returnval){ [ 915.543436] env[64020]: value = "task-407932" [ 915.543436] env[64020]: _type = "Task" [ 915.543436] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 915.555985] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407932, 'name': PowerOnVM_Task} progress is 33%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 915.714951] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-036f8d1c-e40a-48c9-90f1-20c1005f60b2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.722616] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9743cf48-e98b-48b4-a01a-06b11bd77d83 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.753797] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed0d9e45-9748-4129-8506-dc3cbb22795e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.761994] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4e9d0e0-29fb-442e-b1fb-4076cd2a067e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.775847] env[64020]: DEBUG nova.compute.provider_tree [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 915.776501] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 90cd3296e86540ff8ada2049afb25279 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 915.779642] env[64020]: DEBUG nova.compute.manager [req-420d9b6c-16ee-4a97-809b-58ca2b6a4594 req-610cd690-529b-40ae-9dc1-84c2690e41cf service nova] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Received event network-vif-deleted-ca85ae3e-6436-4834-b879-e023f3211066 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 915.786317] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90cd3296e86540ff8ada2049afb25279 [ 915.933575] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 86b5b103177743dfac3544b9ee1dd5c7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 915.969212] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86b5b103177743dfac3544b9ee1dd5c7 [ 915.983279] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Releasing lock "refresh_cache-b1ec9141-54d5-4761-bd37-55a1fc998589" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 915.983807] env[64020]: DEBUG nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 915.983998] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 915.984343] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-54b3bde5-78da-44ab-b04d-daeb0de800d5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.993449] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac40878f-768d-4bf2-ac9f-777269d0399a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.016854] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b1ec9141-54d5-4761-bd37-55a1fc998589 could not be found. [ 916.017104] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 916.017293] env[64020]: INFO nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Took 0.03 seconds to destroy the instance on the hypervisor. [ 916.017539] env[64020]: DEBUG oslo.service.loopingcall [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 916.017773] env[64020]: DEBUG nova.compute.manager [-] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 916.017877] env[64020]: DEBUG nova.network.neutron [-] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 916.033416] env[64020]: DEBUG nova.network.neutron [-] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 916.034002] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b1bd9dce439c497bb65bc4c1b9a57e5a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 916.040678] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1bd9dce439c497bb65bc4c1b9a57e5a [ 916.055770] env[64020]: DEBUG oslo_vmware.api [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407932, 'name': PowerOnVM_Task, 'duration_secs': 0.453223} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 916.056063] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 916.056266] env[64020]: INFO nova.compute.manager [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Took 4.34 seconds to spawn the instance on the hypervisor. [ 916.056560] env[64020]: DEBUG nova.compute.manager [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 916.057315] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83b88779-8b45-4809-b95f-9316d43bdfd2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.065219] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 981044d8c2fc4a70af20961fd8808943 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 916.101311] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 981044d8c2fc4a70af20961fd8808943 [ 916.281615] env[64020]: DEBUG nova.scheduler.client.report [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 916.284074] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 2051612b19634c729a3c169e109150c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 916.295230] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2051612b19634c729a3c169e109150c1 [ 916.439732] env[64020]: ERROR nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 113dd114-5149-4e96-9b3d-599793e830c8, please check neutron logs for more information. [ 916.439732] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 916.439732] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 916.439732] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 916.439732] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 916.439732] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 916.439732] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 916.439732] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 916.439732] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 916.439732] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 916.439732] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 916.439732] env[64020]: ERROR nova.compute.manager raise self.value [ 916.439732] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 916.439732] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 916.439732] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 916.439732] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 916.440669] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 916.440669] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 916.440669] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 113dd114-5149-4e96-9b3d-599793e830c8, please check neutron logs for more information. [ 916.440669] env[64020]: ERROR nova.compute.manager [ 916.440669] env[64020]: Traceback (most recent call last): [ 916.440669] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 916.440669] env[64020]: listener.cb(fileno) [ 916.440669] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 916.440669] env[64020]: result = function(*args, **kwargs) [ 916.440669] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 916.440669] env[64020]: return func(*args, **kwargs) [ 916.440669] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 916.440669] env[64020]: raise e [ 916.440669] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 916.440669] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 916.440669] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 916.440669] env[64020]: created_port_ids = self._update_ports_for_instance( [ 916.440669] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 916.440669] env[64020]: with excutils.save_and_reraise_exception(): [ 916.440669] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 916.440669] env[64020]: self.force_reraise() [ 916.440669] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 916.440669] env[64020]: raise self.value [ 916.440669] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 916.440669] env[64020]: updated_port = self._update_port( [ 916.440669] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 916.440669] env[64020]: _ensure_no_port_binding_failure(port) [ 916.440669] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 916.440669] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 916.441950] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 113dd114-5149-4e96-9b3d-599793e830c8, please check neutron logs for more information. [ 916.441950] env[64020]: Removing descriptor: 16 [ 916.441950] env[64020]: DEBUG nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 916.466837] env[64020]: DEBUG nova.virt.hardware [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 916.467104] env[64020]: DEBUG nova.virt.hardware [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 916.467259] env[64020]: DEBUG nova.virt.hardware [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 916.467438] env[64020]: DEBUG nova.virt.hardware [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 916.467579] env[64020]: DEBUG nova.virt.hardware [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 916.467717] env[64020]: DEBUG nova.virt.hardware [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 916.467918] env[64020]: DEBUG nova.virt.hardware [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 916.468091] env[64020]: DEBUG nova.virt.hardware [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 916.468276] env[64020]: DEBUG nova.virt.hardware [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 916.468456] env[64020]: DEBUG nova.virt.hardware [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 916.468942] env[64020]: DEBUG nova.virt.hardware [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 916.469805] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c07717ef-85e2-49e5-a6d1-c2ea1da9298f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.477473] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f30521d6-8c07-49b5-887a-e442f15650f2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.491905] env[64020]: ERROR nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 113dd114-5149-4e96-9b3d-599793e830c8, please check neutron logs for more information. [ 916.491905] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] Traceback (most recent call last): [ 916.491905] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 916.491905] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] yield resources [ 916.491905] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 916.491905] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] self.driver.spawn(context, instance, image_meta, [ 916.491905] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 916.491905] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] self._vmops.spawn(context, instance, image_meta, injected_files, [ 916.491905] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 916.491905] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] vm_ref = self.build_virtual_machine(instance, [ 916.491905] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] vif_infos = vmwarevif.get_vif_info(self._session, [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] for vif in network_info: [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] return self._sync_wrapper(fn, *args, **kwargs) [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] self.wait() [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] self[:] = self._gt.wait() [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] return self._exit_event.wait() [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 916.492366] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] current.throw(*self._exc) [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] result = function(*args, **kwargs) [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] return func(*args, **kwargs) [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] raise e [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] nwinfo = self.network_api.allocate_for_instance( [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] created_port_ids = self._update_ports_for_instance( [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] with excutils.save_and_reraise_exception(): [ 916.492706] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 916.493035] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] self.force_reraise() [ 916.493035] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 916.493035] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] raise self.value [ 916.493035] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 916.493035] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] updated_port = self._update_port( [ 916.493035] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 916.493035] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] _ensure_no_port_binding_failure(port) [ 916.493035] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 916.493035] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] raise exception.PortBindingFailed(port_id=port['id']) [ 916.493035] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] nova.exception.PortBindingFailed: Binding failed for port 113dd114-5149-4e96-9b3d-599793e830c8, please check neutron logs for more information. [ 916.493035] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] [ 916.493035] env[64020]: INFO nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Terminating instance [ 916.494384] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquiring lock "refresh_cache-5554874d-598f-4276-8778-1da6773be649" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 916.494635] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquired lock "refresh_cache-5554874d-598f-4276-8778-1da6773be649" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 916.494876] env[64020]: DEBUG nova.network.neutron [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 916.495399] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg c3fd6e45fdbe484aa78aaef478d67d18 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 916.503950] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3fd6e45fdbe484aa78aaef478d67d18 [ 916.535700] env[64020]: DEBUG nova.network.neutron [-] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 916.536136] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 4953dfbfd9d1486892a874207f0d48bd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 916.544095] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4953dfbfd9d1486892a874207f0d48bd [ 916.581518] env[64020]: INFO nova.compute.manager [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Took 24.68 seconds to build instance. [ 916.581859] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg c7ddd0ebba3d4010b9d7b47711e52939 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 916.597443] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c7ddd0ebba3d4010b9d7b47711e52939 [ 916.787695] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.382s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 916.788527] env[64020]: DEBUG nova.compute.manager [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 916.791451] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg c2117e66c11143a79049845c21608307 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 916.793126] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.064s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 916.794812] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 5d2e40b7cac04a5caff20e43534b00f1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 916.830314] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2117e66c11143a79049845c21608307 [ 916.830911] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d2e40b7cac04a5caff20e43534b00f1 [ 917.014065] env[64020]: DEBUG nova.network.neutron [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 917.038623] env[64020]: INFO nova.compute.manager [-] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Took 1.02 seconds to deallocate network for instance. [ 917.041400] env[64020]: DEBUG nova.compute.claims [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 917.041601] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 917.083698] env[64020]: DEBUG oslo_concurrency.lockutils [None req-917588d5-f7f4-4af1-aa32-4c8126234e60 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lock "3c5781a8-df02-4b4f-ad3f-c04db921748f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 88.458s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 917.084320] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 3d60ae124adf40c9ac080e469d7d11a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.093182] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d60ae124adf40c9ac080e469d7d11a2 [ 917.098746] env[64020]: DEBUG nova.network.neutron [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 917.099289] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg eb07cda2cd984fc2a58840db9f1b9eb7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.107217] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb07cda2cd984fc2a58840db9f1b9eb7 [ 917.176084] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0c3ab77c-6452-4bcc-b59d-1c6a17d17a58 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 3b33d92d90d94532b6ed4afe34043ae0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.185299] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b33d92d90d94532b6ed4afe34043ae0 [ 917.238903] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg d80238ff04b840179f498ea9c651ec6b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.247773] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d80238ff04b840179f498ea9c651ec6b [ 917.298096] env[64020]: DEBUG nova.compute.utils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 917.298733] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 822dd28a0fd84e9c8362a0d31d5a9d82 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.299663] env[64020]: DEBUG nova.compute.manager [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Not allocating networking since 'none' was specified. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 917.309750] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 822dd28a0fd84e9c8362a0d31d5a9d82 [ 917.539676] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54454b28-da44-47a3-9bca-4e2de5382dc9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.547092] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-282dea34-1118-42c1-a8a5-5280e754d565 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.590768] env[64020]: DEBUG nova.compute.manager [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 917.593478] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 0bb36df7f3bd4cb38bec06efd509964e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.595879] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1906803-56b1-4083-871c-96a9db7b5406 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.601499] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Releasing lock "refresh_cache-5554874d-598f-4276-8778-1da6773be649" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 917.602110] env[64020]: DEBUG nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 917.602385] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 917.605557] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f523ce8f-7729-4f2e-a204-67188ebfc3ac {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.608871] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3cc93dca-3bcd-4151-b171-47cbd872b4ca {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.622332] env[64020]: DEBUG nova.compute.provider_tree [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 917.622799] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 7ef850fbb8b64631abb2282178072d57 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.626212] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0bb36df7f3bd4cb38bec06efd509964e [ 917.628678] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6158c134-d76d-4419-92e0-0f0088aefe9a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.639990] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ef850fbb8b64631abb2282178072d57 [ 917.651361] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5554874d-598f-4276-8778-1da6773be649 could not be found. [ 917.651568] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 917.651745] env[64020]: INFO nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Took 0.05 seconds to destroy the instance on the hypervisor. [ 917.651976] env[64020]: DEBUG oslo.service.loopingcall [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 917.652211] env[64020]: DEBUG nova.compute.manager [-] [instance: 5554874d-598f-4276-8778-1da6773be649] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 917.652312] env[64020]: DEBUG nova.network.neutron [-] [instance: 5554874d-598f-4276-8778-1da6773be649] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 917.664337] env[64020]: DEBUG nova.network.neutron [-] [instance: 5554874d-598f-4276-8778-1da6773be649] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 917.664880] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg db34e07dac2f4ebebdc87b274e2c78ac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.671498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db34e07dac2f4ebebdc87b274e2c78ac [ 917.677714] env[64020]: DEBUG nova.compute.manager [None req-0c3ab77c-6452-4bcc-b59d-1c6a17d17a58 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 917.678615] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95f8663b-5230-4388-8e41-26c6f96c5d12 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.688405] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0c3ab77c-6452-4bcc-b59d-1c6a17d17a58 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 908d9f8870164d3eb424f6409946813f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.708764] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 908d9f8870164d3eb424f6409946813f [ 917.740702] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquiring lock "3c5781a8-df02-4b4f-ad3f-c04db921748f" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 917.741009] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lock "3c5781a8-df02-4b4f-ad3f-c04db921748f" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 917.741330] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquiring lock "3c5781a8-df02-4b4f-ad3f-c04db921748f-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 917.741519] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lock "3c5781a8-df02-4b4f-ad3f-c04db921748f-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 917.741723] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lock "3c5781a8-df02-4b4f-ad3f-c04db921748f-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 917.744054] env[64020]: INFO nova.compute.manager [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Terminating instance [ 917.745798] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquiring lock "refresh_cache-3c5781a8-df02-4b4f-ad3f-c04db921748f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 917.745985] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquired lock "refresh_cache-3c5781a8-df02-4b4f-ad3f-c04db921748f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 917.746178] env[64020]: DEBUG nova.network.neutron [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 917.746614] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 179de5df71ff46758890065f65a65911 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.752323] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 179de5df71ff46758890065f65a65911 [ 917.796942] env[64020]: DEBUG nova.compute.manager [req-822fdad1-1b83-4090-bcb3-8507410052ff req-4686071d-3053-4b02-b365-d40b3c25504f service nova] [instance: 5554874d-598f-4276-8778-1da6773be649] Received event network-changed-113dd114-5149-4e96-9b3d-599793e830c8 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 917.797128] env[64020]: DEBUG nova.compute.manager [req-822fdad1-1b83-4090-bcb3-8507410052ff req-4686071d-3053-4b02-b365-d40b3c25504f service nova] [instance: 5554874d-598f-4276-8778-1da6773be649] Refreshing instance network info cache due to event network-changed-113dd114-5149-4e96-9b3d-599793e830c8. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 917.797308] env[64020]: DEBUG oslo_concurrency.lockutils [req-822fdad1-1b83-4090-bcb3-8507410052ff req-4686071d-3053-4b02-b365-d40b3c25504f service nova] Acquiring lock "refresh_cache-5554874d-598f-4276-8778-1da6773be649" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 917.797430] env[64020]: DEBUG oslo_concurrency.lockutils [req-822fdad1-1b83-4090-bcb3-8507410052ff req-4686071d-3053-4b02-b365-d40b3c25504f service nova] Acquired lock "refresh_cache-5554874d-598f-4276-8778-1da6773be649" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 917.797583] env[64020]: DEBUG nova.network.neutron [req-822fdad1-1b83-4090-bcb3-8507410052ff req-4686071d-3053-4b02-b365-d40b3c25504f service nova] [instance: 5554874d-598f-4276-8778-1da6773be649] Refreshing network info cache for port 113dd114-5149-4e96-9b3d-599793e830c8 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 917.797974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-822fdad1-1b83-4090-bcb3-8507410052ff req-4686071d-3053-4b02-b365-d40b3c25504f service nova] Expecting reply to msg 81fddd1ab3144167b5e98c620ba2279d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.804389] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81fddd1ab3144167b5e98c620ba2279d [ 917.804990] env[64020]: DEBUG nova.compute.manager [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 917.806696] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 67ae399c392c43c9ae670d76bf19b6a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 917.840767] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 67ae399c392c43c9ae670d76bf19b6a2 [ 918.114853] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 918.126780] env[64020]: DEBUG nova.scheduler.client.report [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 918.129245] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 615f57d7adfc48729c30c2e2ead3c819 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 918.141401] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 615f57d7adfc48729c30c2e2ead3c819 [ 918.167007] env[64020]: DEBUG nova.network.neutron [-] [instance: 5554874d-598f-4276-8778-1da6773be649] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 918.167406] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg db254fdc763943c29b8f3e41042021b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 918.175890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db254fdc763943c29b8f3e41042021b3 [ 918.190811] env[64020]: INFO nova.compute.manager [None req-0c3ab77c-6452-4bcc-b59d-1c6a17d17a58 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] instance snapshotting [ 918.191437] env[64020]: DEBUG nova.objects.instance [None req-0c3ab77c-6452-4bcc-b59d-1c6a17d17a58 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lazy-loading 'flavor' on Instance uuid 3c5781a8-df02-4b4f-ad3f-c04db921748f {{(pid=64020) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 918.192036] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0c3ab77c-6452-4bcc-b59d-1c6a17d17a58 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg b95961560ce04e9794f0139bd41acc04 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 918.215284] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b95961560ce04e9794f0139bd41acc04 [ 918.263439] env[64020]: DEBUG nova.network.neutron [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 918.308517] env[64020]: DEBUG nova.network.neutron [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 918.309271] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg a5ff83ca649b4367bbf0b970502dd4bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 918.312954] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg f40b3f889be34eb79642cdfd6e30fec5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 918.316326] env[64020]: DEBUG nova.network.neutron [req-822fdad1-1b83-4090-bcb3-8507410052ff req-4686071d-3053-4b02-b365-d40b3c25504f service nova] [instance: 5554874d-598f-4276-8778-1da6773be649] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 918.321913] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a5ff83ca649b4367bbf0b970502dd4bc [ 918.343658] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f40b3f889be34eb79642cdfd6e30fec5 [ 918.390515] env[64020]: DEBUG nova.network.neutron [req-822fdad1-1b83-4090-bcb3-8507410052ff req-4686071d-3053-4b02-b365-d40b3c25504f service nova] [instance: 5554874d-598f-4276-8778-1da6773be649] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 918.391040] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-822fdad1-1b83-4090-bcb3-8507410052ff req-4686071d-3053-4b02-b365-d40b3c25504f service nova] Expecting reply to msg 023f40fed21a44ba93dec737a7628f1c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 918.398432] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 023f40fed21a44ba93dec737a7628f1c [ 918.632153] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.839s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 918.632821] env[64020]: ERROR nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f581214d-ee3d-44b6-b35c-df009ea243c5, please check neutron logs for more information. [ 918.632821] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Traceback (most recent call last): [ 918.632821] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 918.632821] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] self.driver.spawn(context, instance, image_meta, [ 918.632821] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 918.632821] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] self._vmops.spawn(context, instance, image_meta, injected_files, [ 918.632821] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 918.632821] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] vm_ref = self.build_virtual_machine(instance, [ 918.632821] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 918.632821] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] vif_infos = vmwarevif.get_vif_info(self._session, [ 918.632821] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] for vif in network_info: [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] return self._sync_wrapper(fn, *args, **kwargs) [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] self.wait() [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] self[:] = self._gt.wait() [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] return self._exit_event.wait() [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] current.throw(*self._exc) [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 918.633211] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] result = function(*args, **kwargs) [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] return func(*args, **kwargs) [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] raise e [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] nwinfo = self.network_api.allocate_for_instance( [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] created_port_ids = self._update_ports_for_instance( [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] with excutils.save_and_reraise_exception(): [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] self.force_reraise() [ 918.633548] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 918.633910] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] raise self.value [ 918.633910] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 918.633910] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] updated_port = self._update_port( [ 918.633910] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 918.633910] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] _ensure_no_port_binding_failure(port) [ 918.633910] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 918.633910] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] raise exception.PortBindingFailed(port_id=port['id']) [ 918.633910] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] nova.exception.PortBindingFailed: Binding failed for port f581214d-ee3d-44b6-b35c-df009ea243c5, please check neutron logs for more information. [ 918.633910] env[64020]: ERROR nova.compute.manager [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] [ 918.633910] env[64020]: DEBUG nova.compute.utils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Binding failed for port f581214d-ee3d-44b6-b35c-df009ea243c5, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 918.634755] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.209s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 918.636588] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 9e9ed06a9cda48d1a74759265c9961d7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 918.638046] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Build of instance a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423 was re-scheduled: Binding failed for port f581214d-ee3d-44b6-b35c-df009ea243c5, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 918.638466] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 918.638732] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "refresh_cache-a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 918.638911] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquired lock "refresh_cache-a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 918.639080] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 918.639480] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 132f662359424a45bd0fbf1ff8f9ddcc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 918.645953] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 132f662359424a45bd0fbf1ff8f9ddcc [ 918.669349] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e9ed06a9cda48d1a74759265c9961d7 [ 918.669713] env[64020]: INFO nova.compute.manager [-] [instance: 5554874d-598f-4276-8778-1da6773be649] Took 1.02 seconds to deallocate network for instance. [ 918.672619] env[64020]: DEBUG nova.compute.claims [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 918.672891] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 918.696876] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c219fc9b-8465-4abc-9cbe-651c3f98b4e0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.715676] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b01bf45-c477-4370-8d90-e0c41f1707d4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.723910] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0c3ab77c-6452-4bcc-b59d-1c6a17d17a58 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 058a46048a07480196154fa5d026b89a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 918.753988] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 058a46048a07480196154fa5d026b89a [ 918.815351] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Releasing lock "refresh_cache-3c5781a8-df02-4b4f-ad3f-c04db921748f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 918.815828] env[64020]: DEBUG nova.compute.manager [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 918.816034] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 918.817069] env[64020]: DEBUG nova.compute.manager [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 918.820177] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98431b42-fd40-4c56-93b3-8dfbb71a6ec1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.829428] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 918.829674] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-89652a43-d91c-41a4-a5e3-6e75447f57c9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.838662] env[64020]: DEBUG nova.virt.hardware [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 918.838888] env[64020]: DEBUG nova.virt.hardware [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 918.839042] env[64020]: DEBUG nova.virt.hardware [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 918.839216] env[64020]: DEBUG nova.virt.hardware [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 918.839360] env[64020]: DEBUG nova.virt.hardware [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 918.839502] env[64020]: DEBUG nova.virt.hardware [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 918.839703] env[64020]: DEBUG nova.virt.hardware [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 918.839865] env[64020]: DEBUG nova.virt.hardware [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 918.840085] env[64020]: DEBUG nova.virt.hardware [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 918.840340] env[64020]: DEBUG nova.virt.hardware [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 918.840594] env[64020]: DEBUG nova.virt.hardware [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 918.841381] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0d86e15-304b-49d1-83dd-a891c6fc803a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.845060] env[64020]: DEBUG oslo_vmware.api [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for the task: (returnval){ [ 918.845060] env[64020]: value = "task-407933" [ 918.845060] env[64020]: _type = "Task" [ 918.845060] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 918.851434] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99ec078b-8545-432c-9091-616d0192499b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.857954] env[64020]: DEBUG oslo_vmware.api [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407933, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 918.867216] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 918.872903] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Creating folder: Project (2e01cfe0ef0d432191a81b33a72971be). Parent ref: group-v110249. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 918.873188] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-4ec461f0-78bc-4a93-b960-eee5ede9a232 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.883014] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Created folder: Project (2e01cfe0ef0d432191a81b33a72971be) in parent group-v110249. [ 918.883200] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Creating folder: Instances. Parent ref: group-v110277. {{(pid=64020) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 918.883428] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-0136018c-8259-48dd-a606-8bd997a59352 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.891796] env[64020]: INFO nova.virt.vmwareapi.vm_util [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Created folder: Instances in parent group-v110277. [ 918.892095] env[64020]: DEBUG oslo.service.loopingcall [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 918.892324] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 918.892516] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-2d209d5f-41de-46a3-a5a0-1e48ed9fe4b5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.904902] env[64020]: DEBUG oslo_concurrency.lockutils [req-822fdad1-1b83-4090-bcb3-8507410052ff req-4686071d-3053-4b02-b365-d40b3c25504f service nova] Releasing lock "refresh_cache-5554874d-598f-4276-8778-1da6773be649" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 918.905127] env[64020]: DEBUG nova.compute.manager [req-822fdad1-1b83-4090-bcb3-8507410052ff req-4686071d-3053-4b02-b365-d40b3c25504f service nova] [instance: 5554874d-598f-4276-8778-1da6773be649] Received event network-vif-deleted-113dd114-5149-4e96-9b3d-599793e830c8 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 918.909541] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 918.909541] env[64020]: value = "task-407936" [ 918.909541] env[64020]: _type = "Task" [ 918.909541] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 918.916777] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407936, 'name': CreateVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 919.158275] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 919.226416] env[64020]: DEBUG nova.compute.manager [None req-0c3ab77c-6452-4bcc-b59d-1c6a17d17a58 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Instance disappeared during snapshot {{(pid=64020) _snapshot_instance /opt/stack/nova/nova/compute/manager.py:4494}} [ 919.240779] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 919.242471] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 9762cbb2ce8a42af8ee2ab6e22f6bea2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 919.250910] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9762cbb2ce8a42af8ee2ab6e22f6bea2 [ 919.356488] env[64020]: DEBUG oslo_vmware.api [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407933, 'name': PowerOffVM_Task, 'duration_secs': 0.114567} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 919.357506] env[64020]: DEBUG nova.compute.manager [None req-0c3ab77c-6452-4bcc-b59d-1c6a17d17a58 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Found 0 images (rotation: 2) {{(pid=64020) _rotate_backups /opt/stack/nova/nova/compute/manager.py:4554}} [ 919.357998] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0c3ab77c-6452-4bcc-b59d-1c6a17d17a58 tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg bef80d1644884ed6bb981af1c13f6a5a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 919.362783] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 919.362951] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 919.363554] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-3c74a2a1-1984-417b-868a-8f45159063f6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.368546] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bef80d1644884ed6bb981af1c13f6a5a [ 919.389075] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 919.389278] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Deleting contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 919.389451] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Deleting the datastore file [datastore1] 3c5781a8-df02-4b4f-ad3f-c04db921748f {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 919.389684] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-de40af00-9d2b-4958-91a0-473948bf555c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.395487] env[64020]: DEBUG oslo_vmware.api [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for the task: (returnval){ [ 919.395487] env[64020]: value = "task-407938" [ 919.395487] env[64020]: _type = "Task" [ 919.395487] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 919.405828] env[64020]: DEBUG oslo_vmware.api [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407938, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 919.415081] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37d74314-229f-4e21-ad9a-a0cd368f651e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.421915] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407936, 'name': CreateVM_Task, 'duration_secs': 0.232265} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 919.423591] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 919.424083] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 919.424246] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 919.424555] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 919.425490] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6edbf57a-6cd6-4f0b-a8c6-afdb4fab03f8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.428287] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-6c3cce68-557e-4b13-b2bb-04c0818e9bbd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.433087] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 919.433087] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52a58fd7-7073-9c16-f98f-f9c557c30c36" [ 919.433087] env[64020]: _type = "Task" [ 919.433087] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 919.462357] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-074c7adf-3166-43b9-addc-0e91c2609e4f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.469957] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52a58fd7-7073-9c16-f98f-f9c557c30c36, 'name': SearchDatastore_Task, 'duration_secs': 0.030385} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 919.472042] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 919.472340] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 919.472515] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 919.472665] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 919.472840] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 919.473116] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-938fc0b9-f0e8-4da5-85d8-fbd958614d06 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.475560] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c46531e3-c473-4278-8085-f9a4b7a17e8a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.489044] env[64020]: DEBUG nova.compute.provider_tree [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 919.489564] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 0c61ffb19bd64b6a92b6e6e26e91a173 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 919.491916] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 919.492068] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 919.492773] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-69be8358-797f-4c51-b700-8723697c554b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.497533] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c61ffb19bd64b6a92b6e6e26e91a173 [ 919.498004] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 919.498004] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52502aa8-673b-6e79-e84c-9de1dcaf8571" [ 919.498004] env[64020]: _type = "Task" [ 919.498004] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 919.505484] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52502aa8-673b-6e79-e84c-9de1dcaf8571, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 919.750893] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Releasing lock "refresh_cache-a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 919.751150] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 919.751719] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 919.751719] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 919.765952] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 919.766335] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 8e3de085ddf9421ab31e3a4b0459abff in queue reply_57893177120949e6a93cb88e15cd42b4 [ 919.772908] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8e3de085ddf9421ab31e3a4b0459abff [ 919.906027] env[64020]: DEBUG oslo_vmware.api [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Task: {'id': task-407938, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.099555} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 919.906282] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 919.906457] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Deleted contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 919.906624] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 919.906788] env[64020]: INFO nova.compute.manager [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Took 1.09 seconds to destroy the instance on the hypervisor. [ 919.907011] env[64020]: DEBUG oslo.service.loopingcall [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 919.907190] env[64020]: DEBUG nova.compute.manager [-] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 919.907275] env[64020]: DEBUG nova.network.neutron [-] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 919.921223] env[64020]: DEBUG nova.network.neutron [-] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 919.921696] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0bbd890c73ca421faa200f9f93ace186 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 919.928498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0bbd890c73ca421faa200f9f93ace186 [ 919.992480] env[64020]: DEBUG nova.scheduler.client.report [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 919.994974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg dacd52ff2dff42c5b8cc7025ede1fe9f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 920.010082] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52502aa8-673b-6e79-e84c-9de1dcaf8571, 'name': SearchDatastore_Task, 'duration_secs': 0.007827} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 920.010753] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dacd52ff2dff42c5b8cc7025ede1fe9f [ 920.011885] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-114a076f-66ba-436b-b99d-6128a0847394 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.017565] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 920.017565] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52b481a0-3272-f3ba-69fc-8220bd507633" [ 920.017565] env[64020]: _type = "Task" [ 920.017565] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 920.025251] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52b481a0-3272-f3ba-69fc-8220bd507633, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 920.268620] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 920.269193] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg fc30fa05a0cd4d13855972c1eaf9b80d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 920.277414] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc30fa05a0cd4d13855972c1eaf9b80d [ 920.424334] env[64020]: DEBUG nova.network.neutron [-] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 920.424838] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5cd200ca88464971a4d0a35441483c77 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 920.433380] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5cd200ca88464971a4d0a35441483c77 [ 920.497303] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.862s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 920.498063] env[64020]: ERROR nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 8dc219a1-523c-47f8-b1ec-2151f1c76264, please check neutron logs for more information. [ 920.498063] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Traceback (most recent call last): [ 920.498063] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 920.498063] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] self.driver.spawn(context, instance, image_meta, [ 920.498063] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 920.498063] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 920.498063] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 920.498063] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] vm_ref = self.build_virtual_machine(instance, [ 920.498063] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 920.498063] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] vif_infos = vmwarevif.get_vif_info(self._session, [ 920.498063] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] for vif in network_info: [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] return self._sync_wrapper(fn, *args, **kwargs) [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] self.wait() [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] self[:] = self._gt.wait() [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] return self._exit_event.wait() [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] current.throw(*self._exc) [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 920.498467] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] result = function(*args, **kwargs) [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] return func(*args, **kwargs) [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] raise e [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] nwinfo = self.network_api.allocate_for_instance( [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] created_port_ids = self._update_ports_for_instance( [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] with excutils.save_and_reraise_exception(): [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] self.force_reraise() [ 920.498860] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 920.499178] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] raise self.value [ 920.499178] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 920.499178] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] updated_port = self._update_port( [ 920.499178] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 920.499178] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] _ensure_no_port_binding_failure(port) [ 920.499178] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 920.499178] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] raise exception.PortBindingFailed(port_id=port['id']) [ 920.499178] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] nova.exception.PortBindingFailed: Binding failed for port 8dc219a1-523c-47f8-b1ec-2151f1c76264, please check neutron logs for more information. [ 920.499178] env[64020]: ERROR nova.compute.manager [instance: f4017c56-63b3-420d-91ab-4565d82305fd] [ 920.499178] env[64020]: DEBUG nova.compute.utils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Binding failed for port 8dc219a1-523c-47f8-b1ec-2151f1c76264, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 920.500407] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Build of instance f4017c56-63b3-420d-91ab-4565d82305fd was re-scheduled: Binding failed for port 8dc219a1-523c-47f8-b1ec-2151f1c76264, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 920.500814] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 920.501042] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquiring lock "refresh_cache-f4017c56-63b3-420d-91ab-4565d82305fd" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 920.501263] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Acquired lock "refresh_cache-f4017c56-63b3-420d-91ab-4565d82305fd" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 920.501408] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 920.501820] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 24a8f7b9ea2443eeaaad832132259677 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 920.503301] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.740s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 920.505222] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg ab64956bb8eb495ead858320e42a0705 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 920.508629] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 24a8f7b9ea2443eeaaad832132259677 [ 920.527367] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52b481a0-3272-f3ba-69fc-8220bd507633, 'name': SearchDatastore_Task, 'duration_secs': 0.00841} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 920.527789] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 920.528176] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 77801ae9-89db-4dc7-af03-0646af73b121/77801ae9-89db-4dc7-af03-0646af73b121.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 920.528565] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-9c5c7f70-7ca1-4b53-91f8-7fb62f939859 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.536119] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 920.536119] env[64020]: value = "task-407939" [ 920.536119] env[64020]: _type = "Task" [ 920.536119] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 920.537136] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab64956bb8eb495ead858320e42a0705 [ 920.545678] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407939, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 920.771500] env[64020]: INFO nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423] Took 1.02 seconds to deallocate network for instance. [ 920.773539] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 4aaaa85e721046d68577fec0546c7535 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 920.808476] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4aaaa85e721046d68577fec0546c7535 [ 920.927783] env[64020]: INFO nova.compute.manager [-] [instance: 3c5781a8-df02-4b4f-ad3f-c04db921748f] Took 1.02 seconds to deallocate network for instance. [ 920.933490] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 5d8746e4f8ff4f15a856a99dfd6e0033 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 920.963897] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d8746e4f8ff4f15a856a99dfd6e0033 [ 921.018591] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 921.046511] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407939, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.438888} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 921.048826] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 77801ae9-89db-4dc7-af03-0646af73b121/77801ae9-89db-4dc7-af03-0646af73b121.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 921.049034] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 921.049447] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-69b63e5a-fb60-4163-91da-c3edfd70ceb8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 921.055799] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 921.055799] env[64020]: value = "task-407940" [ 921.055799] env[64020]: _type = "Task" [ 921.055799] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 921.066418] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407940, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 921.123550] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 921.124091] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg b6f519d86460443ebdbdedf4140f05aa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 921.135532] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b6f519d86460443ebdbdedf4140f05aa [ 921.274108] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03c447c0-be7e-4b9f-9419-e948e0844565 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 921.278739] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 794405d537334b4499af07a922a4e00d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 921.284547] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb8918cb-e929-4dce-8328-6882dcbcbadb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 921.318817] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 794405d537334b4499af07a922a4e00d [ 921.319828] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c483a178-3825-4dd1-86a2-e2cc61327a49 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 921.328064] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-525b821d-7f8c-4014-b0fa-1c36e6d695db {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 921.343416] env[64020]: DEBUG nova.compute.provider_tree [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 921.343957] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 3d787568ae8b4a49835a73d91fba5d07 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 921.351317] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d787568ae8b4a49835a73d91fba5d07 [ 921.436928] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 921.568059] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407940, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.057164} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 921.568394] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 921.569208] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba1ec044-31ea-4743-96fe-0f7e31754845 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 921.588305] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Reconfiguring VM instance instance-00000047 to attach disk [datastore1] 77801ae9-89db-4dc7-af03-0646af73b121/77801ae9-89db-4dc7-af03-0646af73b121.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 921.588550] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-e03379e1-1c25-4802-9b9d-7c2f1d73f57a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 921.606854] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 921.606854] env[64020]: value = "task-407941" [ 921.606854] env[64020]: _type = "Task" [ 921.606854] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 921.617504] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407941, 'name': ReconfigVM_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 921.626071] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Releasing lock "refresh_cache-f4017c56-63b3-420d-91ab-4565d82305fd" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 921.626280] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 921.626486] env[64020]: DEBUG nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 921.626669] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 921.640725] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 921.641387] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 4d4124a98eca4cd3a1a0f7fede9e50f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 921.648268] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d4124a98eca4cd3a1a0f7fede9e50f5 [ 921.799658] env[64020]: INFO nova.scheduler.client.report [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Deleted allocations for instance a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423 [ 921.813816] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 4336b874b4464f3db11150b401bc8de2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 921.821270] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4336b874b4464f3db11150b401bc8de2 [ 921.853494] env[64020]: DEBUG nova.scheduler.client.report [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 921.856072] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 469c8689370a4e9ca6ff70d8c3702cf0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 921.871837] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 469c8689370a4e9ca6ff70d8c3702cf0 [ 922.117142] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407941, 'name': ReconfigVM_Task, 'duration_secs': 0.265304} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 922.117418] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Reconfigured VM instance instance-00000047 to attach disk [datastore1] 77801ae9-89db-4dc7-af03-0646af73b121/77801ae9-89db-4dc7-af03-0646af73b121.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 922.118010] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-687deaf3-7e39-4060-9960-ec69ec0049f7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.124081] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 922.124081] env[64020]: value = "task-407942" [ 922.124081] env[64020]: _type = "Task" [ 922.124081] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 922.137404] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407942, 'name': Rename_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 922.143743] env[64020]: DEBUG nova.network.neutron [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 922.144237] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg f0094ee5f3e44170b2917cf6a00deb56 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 922.152520] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f0094ee5f3e44170b2917cf6a00deb56 [ 922.308035] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "a0ed1bf3-5a1d-4c1c-9cef-5c4c967cf423" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 133.353s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 922.308598] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg e523277c53884bfd98c02e04b351a4c9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 922.318096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e523277c53884bfd98c02e04b351a4c9 [ 922.359700] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.856s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 922.360330] env[64020]: ERROR nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a7146506-944d-49f6-bbee-01d781fb3c39, please check neutron logs for more information. [ 922.360330] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Traceback (most recent call last): [ 922.360330] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 922.360330] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] self.driver.spawn(context, instance, image_meta, [ 922.360330] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 922.360330] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 922.360330] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 922.360330] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] vm_ref = self.build_virtual_machine(instance, [ 922.360330] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 922.360330] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] vif_infos = vmwarevif.get_vif_info(self._session, [ 922.360330] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] for vif in network_info: [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] return self._sync_wrapper(fn, *args, **kwargs) [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] self.wait() [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] self[:] = self._gt.wait() [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] return self._exit_event.wait() [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] current.throw(*self._exc) [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 922.360822] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] result = function(*args, **kwargs) [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] return func(*args, **kwargs) [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] raise e [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] nwinfo = self.network_api.allocate_for_instance( [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] created_port_ids = self._update_ports_for_instance( [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] with excutils.save_and_reraise_exception(): [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] self.force_reraise() [ 922.361406] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 922.361946] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] raise self.value [ 922.361946] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 922.361946] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] updated_port = self._update_port( [ 922.361946] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 922.361946] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] _ensure_no_port_binding_failure(port) [ 922.361946] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 922.361946] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] raise exception.PortBindingFailed(port_id=port['id']) [ 922.361946] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] nova.exception.PortBindingFailed: Binding failed for port a7146506-944d-49f6-bbee-01d781fb3c39, please check neutron logs for more information. [ 922.361946] env[64020]: ERROR nova.compute.manager [instance: 5a021861-9784-431d-b717-c7b24fe8525c] [ 922.361946] env[64020]: DEBUG nova.compute.utils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Binding failed for port a7146506-944d-49f6-bbee-01d781fb3c39, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 922.362402] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 15.934s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 922.362402] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 922.362523] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=64020) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 922.362783] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.821s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 922.364994] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 099dd777f6c14407af4af234baafca62 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 922.366418] env[64020]: DEBUG nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Build of instance 5a021861-9784-431d-b717-c7b24fe8525c was re-scheduled: Binding failed for port a7146506-944d-49f6-bbee-01d781fb3c39, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 922.366930] env[64020]: DEBUG nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 922.367163] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Acquiring lock "refresh_cache-5a021861-9784-431d-b717-c7b24fe8525c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 922.367309] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Acquired lock "refresh_cache-5a021861-9784-431d-b717-c7b24fe8525c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 922.367467] env[64020]: DEBUG nova.network.neutron [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 922.367815] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg a231cb4b519c4f3e9feaa7959d80f30e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 922.368972] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d19e396-f7a4-4876-a4eb-143e3947f4fd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.373944] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a231cb4b519c4f3e9feaa7959d80f30e [ 922.379712] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ac6b028-7a55-438b-9d21-44689f54b518 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.394026] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-371fc50f-7986-460c-b06a-c0453e979efd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.396725] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 099dd777f6c14407af4af234baafca62 [ 922.401002] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76620c47-38d7-4382-878c-158adf478f5c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.430150] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181567MB free_disk=120GB free_vcpus=48 pci_devices=None {{(pid=64020) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 922.430418] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 922.634130] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407942, 'name': Rename_Task, 'duration_secs': 0.130342} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 922.634416] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 922.634662] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-e39ced30-23c4-4241-9ab5-49533cbd5d16 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.640126] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 922.640126] env[64020]: value = "task-407943" [ 922.640126] env[64020]: _type = "Task" [ 922.640126] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 922.647397] env[64020]: INFO nova.compute.manager [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] [instance: f4017c56-63b3-420d-91ab-4565d82305fd] Took 1.02 seconds to deallocate network for instance. [ 922.649029] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 725a89785d414171bb31e97e3803c3cf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 922.649952] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407943, 'name': PowerOnVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 922.683430] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 725a89785d414171bb31e97e3803c3cf [ 922.810670] env[64020]: DEBUG nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 922.812843] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg f23b1fa956b2471fb32c1c6b3bd04404 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 922.862776] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f23b1fa956b2471fb32c1c6b3bd04404 [ 922.902831] env[64020]: DEBUG nova.network.neutron [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 922.988396] env[64020]: DEBUG nova.network.neutron [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 922.988396] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 0cf6a2bec2134664a4104073454511ea in queue reply_57893177120949e6a93cb88e15cd42b4 [ 922.996390] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0cf6a2bec2134664a4104073454511ea [ 923.153095] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg 28fe250e0dd1430988fa822690d0f522 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 923.154147] env[64020]: DEBUG oslo_vmware.api [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407943, 'name': PowerOnVM_Task, 'duration_secs': 0.446631} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 923.156758] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 923.157037] env[64020]: INFO nova.compute.manager [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Took 4.34 seconds to spawn the instance on the hypervisor. [ 923.157275] env[64020]: DEBUG nova.compute.manager [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 923.158220] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0320f344-2cf6-46bf-ba4e-71efca6251b7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.165720] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 5000920f97f8498eae26870d66df33d8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 923.194553] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28fe250e0dd1430988fa822690d0f522 [ 923.205383] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e15cd5f4-bab1-41a6-bc58-cf5a02e3d1ec {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.212821] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5000920f97f8498eae26870d66df33d8 [ 923.214045] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40fee39c-e7f8-45fc-b6a8-3e985e178372 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.244036] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c250819-efa7-4dc1-9e70-30c71b924de6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.251366] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de8773cc-ab85-4ce8-865e-f76dd10c16fa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.265135] env[64020]: DEBUG nova.compute.provider_tree [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 923.265760] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg f4e597578b8647f39361ac1b9fcf0e9c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 923.274298] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f4e597578b8647f39361ac1b9fcf0e9c [ 923.331459] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 923.490053] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Releasing lock "refresh_cache-5a021861-9784-431d-b717-c7b24fe8525c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 923.490401] env[64020]: DEBUG nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 923.490673] env[64020]: DEBUG nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 923.490909] env[64020]: DEBUG nova.network.neutron [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 923.513619] env[64020]: DEBUG nova.network.neutron [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 923.514247] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg cebdef4f6755425bb9164fd66cce3550 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 923.522152] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cebdef4f6755425bb9164fd66cce3550 [ 923.672309] env[64020]: INFO nova.scheduler.client.report [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Deleted allocations for instance f4017c56-63b3-420d-91ab-4565d82305fd [ 923.679942] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Expecting reply to msg e7acc9f303fd44988c4c956159aefbac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 923.680758] env[64020]: INFO nova.compute.manager [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Took 23.86 seconds to build instance. [ 923.681039] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 03f3e1d42c084c47b17f714911dc3098 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 923.700179] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e7acc9f303fd44988c4c956159aefbac [ 923.711392] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03f3e1d42c084c47b17f714911dc3098 [ 923.768275] env[64020]: DEBUG nova.scheduler.client.report [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 923.770740] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg dda3efdedc41496ca91b9529347db742 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 923.782939] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dda3efdedc41496ca91b9529347db742 [ 924.017023] env[64020]: DEBUG nova.network.neutron [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 924.017634] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 27a6749aa9e941b4affa90807361fcde in queue reply_57893177120949e6a93cb88e15cd42b4 [ 924.027866] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27a6749aa9e941b4affa90807361fcde [ 924.182536] env[64020]: DEBUG oslo_concurrency.lockutils [None req-799186b1-a36b-4bed-9767-1acd746e1656 tempest-MultipleCreateTestJSON-1764946770 tempest-MultipleCreateTestJSON-1764946770-project-member] Lock "f4017c56-63b3-420d-91ab-4565d82305fd" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 135.182s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 924.182866] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dc7cc6bf-e7cb-4acb-ae36-1f570fd97d56 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "77801ae9-89db-4dc7-af03-0646af73b121" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 90.116s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 924.183412] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 92223f49d7804cd99e9280495cbac720 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 924.184444] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 23ba7d7b486848d6ae059143c8e5c3ef in queue reply_57893177120949e6a93cb88e15cd42b4 [ 924.194369] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92223f49d7804cd99e9280495cbac720 [ 924.201634] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 23ba7d7b486848d6ae059143c8e5c3ef [ 924.274755] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.912s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 924.275395] env[64020]: ERROR nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 2541ed1d-017e-49b6-b5c7-4737c699f975, please check neutron logs for more information. [ 924.275395] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Traceback (most recent call last): [ 924.275395] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 924.275395] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] self.driver.spawn(context, instance, image_meta, [ 924.275395] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 924.275395] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 924.275395] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 924.275395] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] vm_ref = self.build_virtual_machine(instance, [ 924.275395] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 924.275395] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] vif_infos = vmwarevif.get_vif_info(self._session, [ 924.275395] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] for vif in network_info: [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] return self._sync_wrapper(fn, *args, **kwargs) [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] self.wait() [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] self[:] = self._gt.wait() [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] return self._exit_event.wait() [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] result = hub.switch() [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 924.275808] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] return self.greenlet.switch() [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] result = function(*args, **kwargs) [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] return func(*args, **kwargs) [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] raise e [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] nwinfo = self.network_api.allocate_for_instance( [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] created_port_ids = self._update_ports_for_instance( [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] with excutils.save_and_reraise_exception(): [ 924.276221] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 924.276587] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] self.force_reraise() [ 924.276587] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 924.276587] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] raise self.value [ 924.276587] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 924.276587] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] updated_port = self._update_port( [ 924.276587] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 924.276587] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] _ensure_no_port_binding_failure(port) [ 924.276587] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 924.276587] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] raise exception.PortBindingFailed(port_id=port['id']) [ 924.276587] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] nova.exception.PortBindingFailed: Binding failed for port 2541ed1d-017e-49b6-b5c7-4737c699f975, please check neutron logs for more information. [ 924.276587] env[64020]: ERROR nova.compute.manager [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] [ 924.276972] env[64020]: DEBUG nova.compute.utils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Binding failed for port 2541ed1d-017e-49b6-b5c7-4737c699f975, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 924.277224] env[64020]: DEBUG oslo_concurrency.lockutils [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 16.666s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 924.277399] env[64020]: DEBUG nova.objects.instance [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] [instance: 7ba3e477-a789-4beb-90ac-651520836a5d] Trying to apply a migration context that does not seem to be set for this instance {{(pid=64020) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 924.278906] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg bf8da669ec884f419d3583b91e6e3e3b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 924.280190] env[64020]: DEBUG nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Build of instance 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1 was re-scheduled: Binding failed for port 2541ed1d-017e-49b6-b5c7-4737c699f975, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 924.280602] env[64020]: DEBUG nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 924.280823] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "refresh_cache-414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 924.280966] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquired lock "refresh_cache-414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 924.281122] env[64020]: DEBUG nova.network.neutron [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 924.281545] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg f00b2e92f56e4b839d0f998aca9b40c3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 924.287233] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f00b2e92f56e4b839d0f998aca9b40c3 [ 924.310057] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf8da669ec884f419d3583b91e6e3e3b [ 924.520068] env[64020]: INFO nova.compute.manager [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] [instance: 5a021861-9784-431d-b717-c7b24fe8525c] Took 1.03 seconds to deallocate network for instance. [ 924.521986] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 5222879a04154e598db1ebd2b3b98fd8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 924.552833] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5222879a04154e598db1ebd2b3b98fd8 [ 924.686341] env[64020]: DEBUG nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 924.688131] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg dd1a115207c847ef9909411b47fb51ad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 924.691854] env[64020]: DEBUG nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 924.693513] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 4f80fe6f90674e8f8ceefbbab1232ccf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 924.717973] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dd1a115207c847ef9909411b47fb51ad [ 924.724438] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f80fe6f90674e8f8ceefbbab1232ccf [ 924.782901] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 37da0de238044d7a962e6d208fc5a81f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 924.794030] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37da0de238044d7a962e6d208fc5a81f [ 924.798306] env[64020]: DEBUG nova.network.neutron [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 924.880152] env[64020]: DEBUG nova.network.neutron [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 924.880672] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 9dd4f1778eda47cc838a133e163702b9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 924.889443] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9dd4f1778eda47cc838a133e163702b9 [ 925.025975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg d90d46ad74e44edbbd9076cadaa87964 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 925.066192] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d90d46ad74e44edbbd9076cadaa87964 [ 925.208872] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 925.209985] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 925.290913] env[64020]: DEBUG oslo_concurrency.lockutils [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.013s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 925.290913] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-acbf006d-17d8-4c92-b53a-a8d38c14f27d tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg c6666ee314eb4710b153b8bd34e2b0f0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 925.291442] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.099s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 925.293230] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 5c04d10c18164cf28f0e6d197d252ad3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 925.306789] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c6666ee314eb4710b153b8bd34e2b0f0 [ 925.326455] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5c04d10c18164cf28f0e6d197d252ad3 [ 925.382858] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Releasing lock "refresh_cache-414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 925.383099] env[64020]: DEBUG nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 925.384112] env[64020]: DEBUG nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 925.384112] env[64020]: DEBUG nova.network.neutron [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 925.407380] env[64020]: DEBUG nova.network.neutron [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 925.408039] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 5b80cf36a7734534a16ae8bd798c8ed8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 925.415513] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b80cf36a7734534a16ae8bd798c8ed8 [ 925.550712] env[64020]: INFO nova.scheduler.client.report [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Deleted allocations for instance 5a021861-9784-431d-b717-c7b24fe8525c [ 925.557273] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Expecting reply to msg 2f77880669c845f3a2fac608730c7192 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 925.567982] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f77880669c845f3a2fac608730c7192 [ 925.910196] env[64020]: DEBUG nova.network.neutron [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 925.910944] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 576790b601fc488bbf3543d979d691f6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 925.918897] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 576790b601fc488bbf3543d979d691f6 [ 926.058968] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ba6a1b9f-3cd7-42c5-aada-5c048af9fdbf tempest-ServerRescueTestJSON-2091264815 tempest-ServerRescueTestJSON-2091264815-project-member] Lock "5a021861-9784-431d-b717-c7b24fe8525c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 129.675s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 926.059551] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg c282a69db60c4b3f904d3ee5a4dcfb5a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 926.071283] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11334f3b-27c6-4c86-83be-e4aa205fbbc4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 926.081465] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58fa5929-07af-4af8-be30-db9af7ca4fb1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 926.085230] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c282a69db60c4b3f904d3ee5a4dcfb5a [ 926.113252] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-056c8604-ea79-46e5-885d-b8ab38781ca7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 926.123842] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eff33d7a-1c2d-4848-94bc-3b5429b004b6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 926.137740] env[64020]: DEBUG nova.compute.provider_tree [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 926.138275] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 1d735e0ab6fe4811a5e4f73667b4b3c5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 926.148255] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d735e0ab6fe4811a5e4f73667b4b3c5 [ 926.413243] env[64020]: INFO nova.compute.manager [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1] Took 1.03 seconds to deallocate network for instance. [ 926.415007] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 6a8cfbe5b0bb4c189bc0f444b42ad9de in queue reply_57893177120949e6a93cb88e15cd42b4 [ 926.452358] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a8cfbe5b0bb4c189bc0f444b42ad9de [ 926.561987] env[64020]: DEBUG nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 926.563720] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 6f6b3c29f7054696b9c7d675efd4e08e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 926.597217] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f6b3c29f7054696b9c7d675efd4e08e [ 926.641626] env[64020]: DEBUG nova.scheduler.client.report [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 926.644165] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 7a6885db262c464cbef49bd33c988733 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 926.657030] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a6885db262c464cbef49bd33c988733 [ 926.919331] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 361162ffa3a84993ad936638dc306ef6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 926.961696] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 361162ffa3a84993ad936638dc306ef6 [ 927.088134] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 927.147749] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.856s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 927.148420] env[64020]: ERROR nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4728632c-dd93-41a7-8b89-dd53d70a6dd6, please check neutron logs for more information. [ 927.148420] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Traceback (most recent call last): [ 927.148420] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 927.148420] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] self.driver.spawn(context, instance, image_meta, [ 927.148420] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 927.148420] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 927.148420] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 927.148420] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] vm_ref = self.build_virtual_machine(instance, [ 927.148420] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 927.148420] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] vif_infos = vmwarevif.get_vif_info(self._session, [ 927.148420] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] for vif in network_info: [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] return self._sync_wrapper(fn, *args, **kwargs) [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] self.wait() [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] self[:] = self._gt.wait() [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] return self._exit_event.wait() [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] current.throw(*self._exc) [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 927.148782] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] result = function(*args, **kwargs) [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] return func(*args, **kwargs) [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] raise e [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] nwinfo = self.network_api.allocate_for_instance( [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] created_port_ids = self._update_ports_for_instance( [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] with excutils.save_and_reraise_exception(): [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] self.force_reraise() [ 927.149183] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 927.149751] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] raise self.value [ 927.149751] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 927.149751] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] updated_port = self._update_port( [ 927.149751] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 927.149751] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] _ensure_no_port_binding_failure(port) [ 927.149751] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 927.149751] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] raise exception.PortBindingFailed(port_id=port['id']) [ 927.149751] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] nova.exception.PortBindingFailed: Binding failed for port 4728632c-dd93-41a7-8b89-dd53d70a6dd6, please check neutron logs for more information. [ 927.149751] env[64020]: ERROR nova.compute.manager [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] [ 927.149751] env[64020]: DEBUG nova.compute.utils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Binding failed for port 4728632c-dd93-41a7-8b89-dd53d70a6dd6, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 927.150393] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.129s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 927.152066] env[64020]: INFO nova.compute.claims [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 927.153635] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 36199e278c8a4f1680c738496860f52c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 927.154884] env[64020]: DEBUG nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Build of instance a5567127-3d68-4c37-a8aa-03c5579cc18c was re-scheduled: Binding failed for port 4728632c-dd93-41a7-8b89-dd53d70a6dd6, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 927.155300] env[64020]: DEBUG nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 927.155509] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Acquiring lock "refresh_cache-a5567127-3d68-4c37-a8aa-03c5579cc18c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 927.155647] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Acquired lock "refresh_cache-a5567127-3d68-4c37-a8aa-03c5579cc18c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 927.155799] env[64020]: DEBUG nova.network.neutron [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 927.156172] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 2e539c147b2445b1a91858142ecc0cad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 927.163180] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2e539c147b2445b1a91858142ecc0cad [ 927.196798] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36199e278c8a4f1680c738496860f52c [ 927.440579] env[64020]: INFO nova.scheduler.client.report [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Deleted allocations for instance 414c2631-76f5-4a30-9b3e-6f0cefa6f0e1 [ 927.446656] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg ef68bbe6d28e4284b650d1efb2b4fd3f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 927.461018] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef68bbe6d28e4284b650d1efb2b4fd3f [ 927.659400] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 55696d68d80f4ecbb3ffa51565c993e8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 927.672597] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 55696d68d80f4ecbb3ffa51565c993e8 [ 927.675270] env[64020]: DEBUG nova.network.neutron [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 927.754395] env[64020]: DEBUG nova.network.neutron [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 927.754936] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg bc22ebd475764c47bdc2df8b54696a19 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 927.763265] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc22ebd475764c47bdc2df8b54696a19 [ 927.950951] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e0f3b415-5271-4f9b-b1f6-85a4c4a1ac4d tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "414c2631-76f5-4a30-9b3e-6f0cefa6f0e1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 116.833s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 927.950951] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 4bd7240d99bb46d0ae5eff46463a2fa3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 927.961839] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4bd7240d99bb46d0ae5eff46463a2fa3 [ 928.257074] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Releasing lock "refresh_cache-a5567127-3d68-4c37-a8aa-03c5579cc18c" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 928.257302] env[64020]: DEBUG nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 928.257477] env[64020]: DEBUG nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 928.257731] env[64020]: DEBUG nova.network.neutron [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 928.278376] env[64020]: DEBUG nova.network.neutron [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 928.279076] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 42d1cf86aaa1455aae9cd9e9d9f5988e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 928.286283] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42d1cf86aaa1455aae9cd9e9d9f5988e [ 928.380981] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90cabd9f-9f5f-40a6-9fbd-2acb9ff5ff34 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 928.388852] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20e6d884-9b0b-4a7a-83e3-872929d1315f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 928.418155] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c0f9991-cbc8-4185-8697-bdb40b2c07f5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 928.427341] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebdd2b90-50ce-4683-87e8-9501aa098e65 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 928.441669] env[64020]: DEBUG nova.compute.provider_tree [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 928.442165] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 5ae18b2bf9d74494a4cdd885d2f92987 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 928.451770] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ae18b2bf9d74494a4cdd885d2f92987 [ 928.452287] env[64020]: DEBUG nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 928.453918] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 7785a50a5a4748b080100103598e25ba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 928.487984] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7785a50a5a4748b080100103598e25ba [ 928.780808] env[64020]: DEBUG nova.network.neutron [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 928.786387] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 8ed93308339c4d8abdd669c0efccbed2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 928.786387] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "3c6ddf08-d0f3-444b-8249-f9dabeeef87f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 928.786387] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "3c6ddf08-d0f3-444b-8249-f9dabeeef87f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 928.794319] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ed93308339c4d8abdd669c0efccbed2 [ 928.945292] env[64020]: DEBUG nova.scheduler.client.report [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 928.947775] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 4597c6e1dc4a448db201a242ebd126f7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 928.963463] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4597c6e1dc4a448db201a242ebd126f7 [ 928.969983] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 929.288825] env[64020]: INFO nova.compute.manager [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] [instance: a5567127-3d68-4c37-a8aa-03c5579cc18c] Took 1.03 seconds to deallocate network for instance. [ 929.290591] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 07d58deab3124e04a83f04c2b215c302 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 929.324458] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 07d58deab3124e04a83f04c2b215c302 [ 929.449817] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.299s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 929.450343] env[64020]: DEBUG nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 929.452479] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 790d0bab6c674667ae59c4691f328d91 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 929.453499] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 17.030s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 929.453701] env[64020]: DEBUG nova.objects.instance [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lazy-loading 'resources' on Instance uuid 7ba3e477-a789-4beb-90ac-651520836a5d {{(pid=64020) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 929.454002] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 6a88425848584a99aad3443b48c030c3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 929.471979] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a88425848584a99aad3443b48c030c3 [ 929.487008] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 790d0bab6c674667ae59c4691f328d91 [ 929.795437] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg c7983c53d90e42efaa3e7140d67ac57d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 929.825757] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c7983c53d90e42efaa3e7140d67ac57d [ 929.959172] env[64020]: DEBUG nova.compute.utils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 929.960139] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg e8c601b64d8e410ea9160622c4bcaa12 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 929.961552] env[64020]: DEBUG nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 929.962015] env[64020]: DEBUG nova.network.neutron [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 929.977205] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e8c601b64d8e410ea9160622c4bcaa12 [ 930.033459] env[64020]: DEBUG nova.policy [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '045421bc288e438fb9229e8ffaff0686', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cb01421fd6f24fc783dd1e06a9901b94', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 930.190310] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89a23bdd-bb08-4431-b8a1-4a4fe416498c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.198607] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-224d525c-a1a8-444c-a908-9cdf8a35478b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.230555] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29e44a70-f16e-44e6-be74-3324e1d428a6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.238238] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38ab8217-96c2-4baf-9611-f0ba2d558e8f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.252576] env[64020]: DEBUG nova.compute.provider_tree [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 930.253408] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 9bf26733993441d9b84a8724013115ae in queue reply_57893177120949e6a93cb88e15cd42b4 [ 930.262331] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9bf26733993441d9b84a8724013115ae [ 930.310040] env[64020]: DEBUG nova.network.neutron [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Successfully created port: 18a0f809-ed62-4a7d-858d-db5e2199e11d {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 930.337236] env[64020]: INFO nova.scheduler.client.report [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Deleted allocations for instance a5567127-3d68-4c37-a8aa-03c5579cc18c [ 930.343461] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Expecting reply to msg 57df9d83ce1b4a9fa0591c02e18b324f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 930.362187] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 57df9d83ce1b4a9fa0591c02e18b324f [ 930.464584] env[64020]: DEBUG nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 930.466303] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 1aa9a3c5248f43c88665e5a993506f31 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 930.496977] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1aa9a3c5248f43c88665e5a993506f31 [ 930.756193] env[64020]: DEBUG nova.scheduler.client.report [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 930.758700] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 80ca66f76c1f43ba911fd407355facf4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 930.771257] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80ca66f76c1f43ba911fd407355facf4 [ 930.845127] env[64020]: DEBUG oslo_concurrency.lockutils [None req-465d0757-aadf-4a85-923c-5922cc40f330 tempest-ServerAddressesTestJSON-1677727497 tempest-ServerAddressesTestJSON-1677727497-project-member] Lock "a5567127-3d68-4c37-a8aa-03c5579cc18c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 105.771s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 930.845778] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 04b92f626ac64bd3bc30498cf8fddfc9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 930.858630] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 04b92f626ac64bd3bc30498cf8fddfc9 [ 930.970933] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg f7a5cb6a1a2040a2860b13d15c1ca291 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 931.001781] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7a5cb6a1a2040a2860b13d15c1ca291 [ 931.009882] env[64020]: DEBUG nova.compute.manager [req-9d43ec51-2f22-488f-85b8-9b13b3bf7f57 req-5741a3f0-bd95-4a92-8b54-c6c82403b03e service nova] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Received event network-changed-18a0f809-ed62-4a7d-858d-db5e2199e11d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 931.010253] env[64020]: DEBUG nova.compute.manager [req-9d43ec51-2f22-488f-85b8-9b13b3bf7f57 req-5741a3f0-bd95-4a92-8b54-c6c82403b03e service nova] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Refreshing instance network info cache due to event network-changed-18a0f809-ed62-4a7d-858d-db5e2199e11d. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 931.010537] env[64020]: DEBUG oslo_concurrency.lockutils [req-9d43ec51-2f22-488f-85b8-9b13b3bf7f57 req-5741a3f0-bd95-4a92-8b54-c6c82403b03e service nova] Acquiring lock "refresh_cache-186cd540-6bb2-407f-9adb-9ed841656713" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 931.010801] env[64020]: DEBUG oslo_concurrency.lockutils [req-9d43ec51-2f22-488f-85b8-9b13b3bf7f57 req-5741a3f0-bd95-4a92-8b54-c6c82403b03e service nova] Acquired lock "refresh_cache-186cd540-6bb2-407f-9adb-9ed841656713" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 931.010979] env[64020]: DEBUG nova.network.neutron [req-9d43ec51-2f22-488f-85b8-9b13b3bf7f57 req-5741a3f0-bd95-4a92-8b54-c6c82403b03e service nova] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Refreshing network info cache for port 18a0f809-ed62-4a7d-858d-db5e2199e11d {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 931.011552] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-9d43ec51-2f22-488f-85b8-9b13b3bf7f57 req-5741a3f0-bd95-4a92-8b54-c6c82403b03e service nova] Expecting reply to msg fdd4816740654d2bb6f7d4867afbb3d1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 931.017937] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fdd4816740654d2bb6f7d4867afbb3d1 [ 931.158039] env[64020]: ERROR nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 18a0f809-ed62-4a7d-858d-db5e2199e11d, please check neutron logs for more information. [ 931.158039] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 931.158039] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 931.158039] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 931.158039] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 931.158039] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 931.158039] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 931.158039] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 931.158039] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 931.158039] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 931.158039] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 931.158039] env[64020]: ERROR nova.compute.manager raise self.value [ 931.158039] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 931.158039] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 931.158039] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 931.158039] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 931.158454] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 931.158454] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 931.158454] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 18a0f809-ed62-4a7d-858d-db5e2199e11d, please check neutron logs for more information. [ 931.158454] env[64020]: ERROR nova.compute.manager [ 931.158454] env[64020]: Traceback (most recent call last): [ 931.158454] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 931.158454] env[64020]: listener.cb(fileno) [ 931.158454] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 931.158454] env[64020]: result = function(*args, **kwargs) [ 931.158454] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 931.158454] env[64020]: return func(*args, **kwargs) [ 931.158454] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 931.158454] env[64020]: raise e [ 931.158454] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 931.158454] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 931.158454] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 931.158454] env[64020]: created_port_ids = self._update_ports_for_instance( [ 931.158454] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 931.158454] env[64020]: with excutils.save_and_reraise_exception(): [ 931.158454] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 931.158454] env[64020]: self.force_reraise() [ 931.158454] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 931.158454] env[64020]: raise self.value [ 931.158454] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 931.158454] env[64020]: updated_port = self._update_port( [ 931.158454] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 931.158454] env[64020]: _ensure_no_port_binding_failure(port) [ 931.158454] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 931.158454] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 931.159174] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 18a0f809-ed62-4a7d-858d-db5e2199e11d, please check neutron logs for more information. [ 931.159174] env[64020]: Removing descriptor: 18 [ 931.261076] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.807s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 931.263434] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.222s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 931.265278] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 2dd837064c62490f9db901f26603844b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 931.292529] env[64020]: INFO nova.scheduler.client.report [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Deleted allocations for instance 7ba3e477-a789-4beb-90ac-651520836a5d [ 931.295532] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 3167148464d048fc8373878e9465807a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 931.303999] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2dd837064c62490f9db901f26603844b [ 931.336380] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3167148464d048fc8373878e9465807a [ 931.351498] env[64020]: DEBUG nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 931.353373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg a0bce4c3a1df4728b267ed4e51acb421 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 931.386186] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a0bce4c3a1df4728b267ed4e51acb421 [ 931.473802] env[64020]: DEBUG nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 931.498981] env[64020]: DEBUG nova.virt.hardware [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 931.499248] env[64020]: DEBUG nova.virt.hardware [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 931.499413] env[64020]: DEBUG nova.virt.hardware [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 931.499596] env[64020]: DEBUG nova.virt.hardware [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 931.499739] env[64020]: DEBUG nova.virt.hardware [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 931.499881] env[64020]: DEBUG nova.virt.hardware [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 931.500113] env[64020]: DEBUG nova.virt.hardware [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 931.500277] env[64020]: DEBUG nova.virt.hardware [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 931.500440] env[64020]: DEBUG nova.virt.hardware [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 931.500594] env[64020]: DEBUG nova.virt.hardware [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 931.500758] env[64020]: DEBUG nova.virt.hardware [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 931.501638] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9bd75cb-c617-4543-a683-9c6d43741706 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.509829] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5adf30fb-5e50-4ce6-9975-a9341efd438d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.524714] env[64020]: ERROR nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 18a0f809-ed62-4a7d-858d-db5e2199e11d, please check neutron logs for more information. [ 931.524714] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Traceback (most recent call last): [ 931.524714] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 931.524714] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] yield resources [ 931.524714] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 931.524714] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] self.driver.spawn(context, instance, image_meta, [ 931.524714] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 931.524714] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] self._vmops.spawn(context, instance, image_meta, injected_files, [ 931.524714] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 931.524714] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] vm_ref = self.build_virtual_machine(instance, [ 931.524714] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] vif_infos = vmwarevif.get_vif_info(self._session, [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] for vif in network_info: [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] return self._sync_wrapper(fn, *args, **kwargs) [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] self.wait() [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] self[:] = self._gt.wait() [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] return self._exit_event.wait() [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 931.525074] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] current.throw(*self._exc) [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] result = function(*args, **kwargs) [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] return func(*args, **kwargs) [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] raise e [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] nwinfo = self.network_api.allocate_for_instance( [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] created_port_ids = self._update_ports_for_instance( [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] with excutils.save_and_reraise_exception(): [ 931.525448] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 931.525756] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] self.force_reraise() [ 931.525756] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 931.525756] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] raise self.value [ 931.525756] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 931.525756] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] updated_port = self._update_port( [ 931.525756] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 931.525756] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] _ensure_no_port_binding_failure(port) [ 931.525756] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 931.525756] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] raise exception.PortBindingFailed(port_id=port['id']) [ 931.525756] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] nova.exception.PortBindingFailed: Binding failed for port 18a0f809-ed62-4a7d-858d-db5e2199e11d, please check neutron logs for more information. [ 931.525756] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] [ 931.525756] env[64020]: INFO nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Terminating instance [ 931.527074] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquiring lock "refresh_cache-186cd540-6bb2-407f-9adb-9ed841656713" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 931.529583] env[64020]: DEBUG nova.network.neutron [req-9d43ec51-2f22-488f-85b8-9b13b3bf7f57 req-5741a3f0-bd95-4a92-8b54-c6c82403b03e service nova] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 931.609512] env[64020]: DEBUG nova.network.neutron [req-9d43ec51-2f22-488f-85b8-9b13b3bf7f57 req-5741a3f0-bd95-4a92-8b54-c6c82403b03e service nova] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 931.609512] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-9d43ec51-2f22-488f-85b8-9b13b3bf7f57 req-5741a3f0-bd95-4a92-8b54-c6c82403b03e service nova] Expecting reply to msg 6bb692c7c8d24a9eb48b491e16c296af in queue reply_57893177120949e6a93cb88e15cd42b4 [ 931.617166] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6bb692c7c8d24a9eb48b491e16c296af [ 931.801335] env[64020]: DEBUG oslo_concurrency.lockutils [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Lock "7ba3e477-a789-4beb-90ac-651520836a5d" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 23.067s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 931.802051] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-afac5783-9119-4652-ae68-1407453519bf tempest-ServerShowV257Test-1067885230 tempest-ServerShowV257Test-1067885230-project-member] Expecting reply to msg 0b7ecb10221247dd995a48a6238d8549 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 931.815935] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b7ecb10221247dd995a48a6238d8549 [ 931.868480] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 931.970311] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4602f04-37a6-42d4-8afa-fdc1eaddcd86 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.981226] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9d7ade2-b026-405e-a4eb-11371db3f80d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.013898] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a913e5fc-c971-4f5f-bcf3-a830c889cf56 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.023225] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bed5cd8b-7337-49c8-ab38-dc2de56450b7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.039503] env[64020]: DEBUG nova.compute.provider_tree [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 932.040137] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 802dffc8d41e482abc95d3da1cc6526c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 932.050555] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 802dffc8d41e482abc95d3da1cc6526c [ 932.111478] env[64020]: DEBUG oslo_concurrency.lockutils [req-9d43ec51-2f22-488f-85b8-9b13b3bf7f57 req-5741a3f0-bd95-4a92-8b54-c6c82403b03e service nova] Releasing lock "refresh_cache-186cd540-6bb2-407f-9adb-9ed841656713" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 932.111912] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquired lock "refresh_cache-186cd540-6bb2-407f-9adb-9ed841656713" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 932.112128] env[64020]: DEBUG nova.network.neutron [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 932.112574] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg ff35bbdd55d746a9a1447f81d7d58ba4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 932.119678] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff35bbdd55d746a9a1447f81d7d58ba4 [ 932.550730] env[64020]: DEBUG nova.scheduler.client.report [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 932.550730] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 0a23100b53dd404fa4990f43744af227 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 932.573079] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a23100b53dd404fa4990f43744af227 [ 932.633000] env[64020]: DEBUG nova.network.neutron [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 932.720161] env[64020]: DEBUG nova.network.neutron [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 932.720697] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 73e86c9456e24edfbfd7297ee4bdfc47 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 932.732127] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 73e86c9456e24edfbfd7297ee4bdfc47 [ 933.035540] env[64020]: DEBUG nova.compute.manager [req-13420e56-7563-48a0-9236-001d3467a3f1 req-d933dbd8-ff49-4966-a40a-28d787fa887e service nova] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Received event network-vif-deleted-18a0f809-ed62-4a7d-858d-db5e2199e11d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 933.053837] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.790s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 933.054442] env[64020]: ERROR nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ca85ae3e-6436-4834-b879-e023f3211066, please check neutron logs for more information. [ 933.054442] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Traceback (most recent call last): [ 933.054442] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 933.054442] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] self.driver.spawn(context, instance, image_meta, [ 933.054442] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 933.054442] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] self._vmops.spawn(context, instance, image_meta, injected_files, [ 933.054442] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 933.054442] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] vm_ref = self.build_virtual_machine(instance, [ 933.054442] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 933.054442] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] vif_infos = vmwarevif.get_vif_info(self._session, [ 933.054442] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] for vif in network_info: [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] return self._sync_wrapper(fn, *args, **kwargs) [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] self.wait() [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] self[:] = self._gt.wait() [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] return self._exit_event.wait() [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] current.throw(*self._exc) [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 933.054776] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] result = function(*args, **kwargs) [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] return func(*args, **kwargs) [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] raise e [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] nwinfo = self.network_api.allocate_for_instance( [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] created_port_ids = self._update_ports_for_instance( [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] with excutils.save_and_reraise_exception(): [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] self.force_reraise() [ 933.055150] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 933.055514] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] raise self.value [ 933.055514] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 933.055514] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] updated_port = self._update_port( [ 933.055514] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 933.055514] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] _ensure_no_port_binding_failure(port) [ 933.055514] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 933.055514] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] raise exception.PortBindingFailed(port_id=port['id']) [ 933.055514] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] nova.exception.PortBindingFailed: Binding failed for port ca85ae3e-6436-4834-b879-e023f3211066, please check neutron logs for more information. [ 933.055514] env[64020]: ERROR nova.compute.manager [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] [ 933.055514] env[64020]: DEBUG nova.compute.utils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Binding failed for port ca85ae3e-6436-4834-b879-e023f3211066, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 933.056355] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.942s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 933.057769] env[64020]: INFO nova.compute.claims [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 933.059394] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 2b8f583e876e4c36936cec1c3226f0e6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 933.060590] env[64020]: DEBUG nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Build of instance b1ec9141-54d5-4761-bd37-55a1fc998589 was re-scheduled: Binding failed for port ca85ae3e-6436-4834-b879-e023f3211066, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 933.060995] env[64020]: DEBUG nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 933.061254] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquiring lock "refresh_cache-b1ec9141-54d5-4761-bd37-55a1fc998589" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 933.061407] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquired lock "refresh_cache-b1ec9141-54d5-4761-bd37-55a1fc998589" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 933.061567] env[64020]: DEBUG nova.network.neutron [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 933.061916] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 3c3573000470443580199d5389901db0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 933.068112] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c3573000470443580199d5389901db0 [ 933.099619] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b8f583e876e4c36936cec1c3226f0e6 [ 933.223117] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Releasing lock "refresh_cache-186cd540-6bb2-407f-9adb-9ed841656713" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 933.223530] env[64020]: DEBUG nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 933.223699] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 933.224327] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3d3e2008-7518-409f-aa64-b9c4059b989e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.233330] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-507790b1-1957-4c8b-be7e-011d7fbed126 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.254186] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 186cd540-6bb2-407f-9adb-9ed841656713 could not be found. [ 933.254379] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 933.254553] env[64020]: INFO nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Took 0.03 seconds to destroy the instance on the hypervisor. [ 933.254778] env[64020]: DEBUG oslo.service.loopingcall [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 933.254987] env[64020]: DEBUG nova.compute.manager [-] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 933.255078] env[64020]: DEBUG nova.network.neutron [-] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 933.268425] env[64020]: DEBUG nova.network.neutron [-] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 933.268853] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3c8dcaf5210a4046ba4d5d1a4183c14b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 933.275774] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c8dcaf5210a4046ba4d5d1a4183c14b [ 933.565532] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg ace3e013547648d6bad99e3efa0abd1f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 933.573671] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ace3e013547648d6bad99e3efa0abd1f [ 933.581715] env[64020]: DEBUG nova.network.neutron [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 933.654196] env[64020]: DEBUG nova.network.neutron [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 933.654779] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg b9d72cd3a98f4ea180427ccc603e249f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 933.662988] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9d72cd3a98f4ea180427ccc603e249f [ 933.771159] env[64020]: DEBUG nova.network.neutron [-] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 933.771700] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 61123807791149fb8e454bd46b41b3ba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 933.780009] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 61123807791149fb8e454bd46b41b3ba [ 934.157149] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Releasing lock "refresh_cache-b1ec9141-54d5-4761-bd37-55a1fc998589" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 934.157392] env[64020]: DEBUG nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 934.157595] env[64020]: DEBUG nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 934.157701] env[64020]: DEBUG nova.network.neutron [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 934.176800] env[64020]: DEBUG nova.network.neutron [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 934.177393] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg dbe56de1684c4462b72197b3f7120754 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 934.183975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dbe56de1684c4462b72197b3f7120754 [ 934.246446] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98fe3b86-c4d1-4cb3-82c0-346dc351e8e6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.255425] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c5a3791-af05-4ccb-a39f-def092a9b08a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.283441] env[64020]: INFO nova.compute.manager [-] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Took 1.03 seconds to deallocate network for instance. [ 934.285576] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ef9afde-174c-4d86-9ff9-72518d6c5379 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.288031] env[64020]: DEBUG nova.compute.claims [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 934.288218] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 934.293336] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e61ce495-a362-416d-a1d7-32fd00a7404c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.306148] env[64020]: DEBUG nova.compute.provider_tree [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 934.306601] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 3772030119794ae7b45e5111572931c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 934.314011] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3772030119794ae7b45e5111572931c8 [ 934.679787] env[64020]: DEBUG nova.network.neutron [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 934.680343] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 91c2583def674c1ab70264354cce94ae in queue reply_57893177120949e6a93cb88e15cd42b4 [ 934.688414] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91c2583def674c1ab70264354cce94ae [ 934.809261] env[64020]: DEBUG nova.scheduler.client.report [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 934.811941] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 0a06aa9fceba4fb795d253e11a812787 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 934.823899] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a06aa9fceba4fb795d253e11a812787 [ 935.183321] env[64020]: INFO nova.compute.manager [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: b1ec9141-54d5-4761-bd37-55a1fc998589] Took 1.03 seconds to deallocate network for instance. [ 935.188278] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 7db9641bfddd4fc9ad61ab2c8bb3655a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 935.243620] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7db9641bfddd4fc9ad61ab2c8bb3655a [ 935.314202] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.258s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 935.314742] env[64020]: DEBUG nova.compute.manager [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 935.316777] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg e75c8d6fd6e54b9981ccee21e7c59447 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 935.318287] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.645s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 935.320310] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg a0048f8b030d4c05a7c0df3d69edca8d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 935.345009] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e75c8d6fd6e54b9981ccee21e7c59447 [ 935.350719] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a0048f8b030d4c05a7c0df3d69edca8d [ 935.690075] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 8cd99e3842a2474fa3610d549bb72615 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 935.727981] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8cd99e3842a2474fa3610d549bb72615 [ 935.820249] env[64020]: DEBUG nova.compute.utils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 935.820876] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg e0ce44316a3d471d86ea1e34ff2e2b00 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 935.822336] env[64020]: DEBUG nova.compute.manager [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Not allocating networking since 'none' was specified. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 935.837460] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e0ce44316a3d471d86ea1e34ff2e2b00 [ 936.010530] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db77d4b5-ad6e-4c4a-8456-6f8e362a8f67 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 936.017679] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-839965db-29a3-4023-9b42-91ae65264b28 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 936.046523] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ef7cb1a-a306-4fb1-b2f2-991e9e58e855 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 936.053233] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dffbf9eb-cdbe-4ee5-af78-e3dd7e5f3a68 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 936.065527] env[64020]: DEBUG nova.compute.provider_tree [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 936.066005] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 0d56d589be8c46a8a926fbf3bd594e86 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 936.073709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0d56d589be8c46a8a926fbf3bd594e86 [ 936.211936] env[64020]: INFO nova.scheduler.client.report [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Deleted allocations for instance b1ec9141-54d5-4761-bd37-55a1fc998589 [ 936.224058] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 684978841e4146ed8d8f342eeae16d18 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 936.240624] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 684978841e4146ed8d8f342eeae16d18 [ 936.323078] env[64020]: DEBUG nova.compute.manager [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 936.327029] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 43460817b19d49789f67948957c30cff in queue reply_57893177120949e6a93cb88e15cd42b4 [ 936.358492] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43460817b19d49789f67948957c30cff [ 936.569639] env[64020]: DEBUG nova.scheduler.client.report [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 936.572579] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 86161c7ac50f463daa67fcf41d0c4bb3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 936.582974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86161c7ac50f463daa67fcf41d0c4bb3 [ 936.725852] env[64020]: DEBUG oslo_concurrency.lockutils [None req-38a5e484-d7ad-4623-a261-58948e34a593 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "b1ec9141-54d5-4761-bd37-55a1fc998589" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 106.137s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 936.726432] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 9307918689bf4e5280e89e3fa9f6ee70 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 936.742983] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9307918689bf4e5280e89e3fa9f6ee70 [ 936.832356] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg f15f29f7fc0c46228dc38a86058e1387 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 936.862626] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f15f29f7fc0c46228dc38a86058e1387 [ 937.075788] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.757s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 937.076439] env[64020]: ERROR nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 113dd114-5149-4e96-9b3d-599793e830c8, please check neutron logs for more information. [ 937.076439] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] Traceback (most recent call last): [ 937.076439] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 937.076439] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] self.driver.spawn(context, instance, image_meta, [ 937.076439] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 937.076439] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] self._vmops.spawn(context, instance, image_meta, injected_files, [ 937.076439] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 937.076439] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] vm_ref = self.build_virtual_machine(instance, [ 937.076439] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 937.076439] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] vif_infos = vmwarevif.get_vif_info(self._session, [ 937.076439] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] for vif in network_info: [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] return self._sync_wrapper(fn, *args, **kwargs) [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] self.wait() [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] self[:] = self._gt.wait() [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] return self._exit_event.wait() [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] current.throw(*self._exc) [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 937.076734] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] result = function(*args, **kwargs) [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] return func(*args, **kwargs) [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] raise e [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] nwinfo = self.network_api.allocate_for_instance( [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] created_port_ids = self._update_ports_for_instance( [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] with excutils.save_and_reraise_exception(): [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] self.force_reraise() [ 937.077091] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 937.077407] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] raise self.value [ 937.077407] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 937.077407] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] updated_port = self._update_port( [ 937.077407] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 937.077407] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] _ensure_no_port_binding_failure(port) [ 937.077407] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 937.077407] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] raise exception.PortBindingFailed(port_id=port['id']) [ 937.077407] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] nova.exception.PortBindingFailed: Binding failed for port 113dd114-5149-4e96-9b3d-599793e830c8, please check neutron logs for more information. [ 937.077407] env[64020]: ERROR nova.compute.manager [instance: 5554874d-598f-4276-8778-1da6773be649] [ 937.077407] env[64020]: DEBUG nova.compute.utils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Binding failed for port 113dd114-5149-4e96-9b3d-599793e830c8, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 937.078516] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 15.642s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 937.078612] env[64020]: DEBUG nova.objects.instance [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lazy-loading 'resources' on Instance uuid 3c5781a8-df02-4b4f-ad3f-c04db921748f {{(pid=64020) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 937.078943] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 2b10501ec6dd40e0a8a16f2411db4699 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 937.080596] env[64020]: DEBUG nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Build of instance 5554874d-598f-4276-8778-1da6773be649 was re-scheduled: Binding failed for port 113dd114-5149-4e96-9b3d-599793e830c8, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 937.081133] env[64020]: DEBUG nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 937.081419] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquiring lock "refresh_cache-5554874d-598f-4276-8778-1da6773be649" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 937.081562] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Acquired lock "refresh_cache-5554874d-598f-4276-8778-1da6773be649" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 937.081713] env[64020]: DEBUG nova.network.neutron [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 937.082086] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg fa5d758c35e3426790a1829006713829 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 937.087805] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b10501ec6dd40e0a8a16f2411db4699 [ 937.090706] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fa5d758c35e3426790a1829006713829 [ 937.228085] env[64020]: DEBUG nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 937.229985] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 3415364cf89d46d7a62321f3b970466a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 937.266107] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3415364cf89d46d7a62321f3b970466a [ 937.335748] env[64020]: DEBUG nova.compute.manager [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 937.365449] env[64020]: DEBUG nova.virt.hardware [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 937.365699] env[64020]: DEBUG nova.virt.hardware [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 937.365852] env[64020]: DEBUG nova.virt.hardware [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 937.366029] env[64020]: DEBUG nova.virt.hardware [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 937.366175] env[64020]: DEBUG nova.virt.hardware [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 937.366324] env[64020]: DEBUG nova.virt.hardware [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 937.366527] env[64020]: DEBUG nova.virt.hardware [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 937.366681] env[64020]: DEBUG nova.virt.hardware [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 937.366841] env[64020]: DEBUG nova.virt.hardware [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 937.366994] env[64020]: DEBUG nova.virt.hardware [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 937.367264] env[64020]: DEBUG nova.virt.hardware [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 937.368130] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5d47c15-973f-44dc-88f6-aaff67e3ca98 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.375874] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f68b027e-0145-407e-9397-e308102c0056 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.388925] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 937.394296] env[64020]: DEBUG oslo.service.loopingcall [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 937.394513] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 937.394709] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-331bef65-35c4-4a2b-9be4-e858218acbb9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.410793] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 937.410793] env[64020]: value = "task-407944" [ 937.410793] env[64020]: _type = "Task" [ 937.410793] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 937.417743] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407944, 'name': CreateVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 937.605380] env[64020]: DEBUG nova.network.neutron [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 937.695747] env[64020]: DEBUG nova.network.neutron [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 937.696399] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 124ba7b115eb472fbcb56ee43a0f2343 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 937.705479] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 124ba7b115eb472fbcb56ee43a0f2343 [ 937.746363] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 937.769655] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-728f4e38-4a1a-4c75-a3ee-892da4b0d002 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.777208] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4df00cf4-89f7-473c-b425-cc49b817a3a7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.810203] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-510827d3-a021-4311-b990-3c98a95c8a76 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.817534] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4040e132-f67e-45e3-a08f-d8e40bf33b88 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.832761] env[64020]: DEBUG nova.compute.provider_tree [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 937.833411] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 51f7289617f547afadba8e1df3b34174 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 937.840834] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51f7289617f547afadba8e1df3b34174 [ 937.921281] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407944, 'name': CreateVM_Task, 'duration_secs': 0.264236} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 937.921461] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 937.921894] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 937.922051] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 937.922693] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 937.922762] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-e0fa554e-1577-47a1-92ce-44eeea53fe45 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.927397] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 937.927397] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52f5db02-a0db-0b46-aec7-1165999636c5" [ 937.927397] env[64020]: _type = "Task" [ 937.927397] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 937.935114] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52f5db02-a0db-0b46-aec7-1165999636c5, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 938.199309] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Releasing lock "refresh_cache-5554874d-598f-4276-8778-1da6773be649" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 938.199612] env[64020]: DEBUG nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 938.199799] env[64020]: DEBUG nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 938.199970] env[64020]: DEBUG nova.network.neutron [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 938.214938] env[64020]: DEBUG nova.network.neutron [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 938.215507] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 54b6723818904e44875c814c046eb5d0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 938.222521] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54b6723818904e44875c814c046eb5d0 [ 938.336898] env[64020]: DEBUG nova.scheduler.client.report [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 938.339400] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg cd401ab6416749f8a849c06c4d1854eb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 938.351537] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd401ab6416749f8a849c06c4d1854eb [ 938.437949] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52f5db02-a0db-0b46-aec7-1165999636c5, 'name': SearchDatastore_Task, 'duration_secs': 0.009361} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 938.438259] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 938.438494] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 938.438717] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 938.438859] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 938.439050] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 938.439339] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-20d74b4f-f98c-45e8-9edf-fa6721b27265 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 938.447730] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 938.447847] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 938.448549] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5225ba0d-0d86-4bd3-ab03-7b55409377a5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 938.453298] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 938.453298] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52211fb4-f12c-e189-4185-7b328e1028e6" [ 938.453298] env[64020]: _type = "Task" [ 938.453298] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 938.460222] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52211fb4-f12c-e189-4185-7b328e1028e6, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 938.718036] env[64020]: DEBUG nova.network.neutron [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 938.718553] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 6b13da525c00441383d94cf66d4507ed in queue reply_57893177120949e6a93cb88e15cd42b4 [ 938.726674] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6b13da525c00441383d94cf66d4507ed [ 938.843000] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.764s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 938.845747] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 16.415s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 938.846599] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 1f07c61d0c61411188c49986fd26ee90 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 938.861238] env[64020]: INFO nova.scheduler.client.report [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Deleted allocations for instance 3c5781a8-df02-4b4f-ad3f-c04db921748f [ 938.864517] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 80d4683f28df4995bd48d63029e9ac40 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 938.869037] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f07c61d0c61411188c49986fd26ee90 [ 938.903897] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80d4683f28df4995bd48d63029e9ac40 [ 938.963354] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52211fb4-f12c-e189-4185-7b328e1028e6, 'name': SearchDatastore_Task, 'duration_secs': 0.007569} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 938.964123] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b12ec301-4e0b-402d-8d24-9d07377eacb5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 938.969222] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 938.969222] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52be5ca5-860c-eda9-ad8f-2284f2ea6904" [ 938.969222] env[64020]: _type = "Task" [ 938.969222] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 938.977083] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52be5ca5-860c-eda9-ad8f-2284f2ea6904, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 939.221227] env[64020]: INFO nova.compute.manager [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] [instance: 5554874d-598f-4276-8778-1da6773be649] Took 1.02 seconds to deallocate network for instance. [ 939.223005] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 4079e55b87a245dc9711d5b0e5fc88e2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 939.252827] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4079e55b87a245dc9711d5b0e5fc88e2 [ 939.352039] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 851c101fffd44405a741ed575bc339b5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 939.360874] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 851c101fffd44405a741ed575bc339b5 [ 939.371008] env[64020]: DEBUG oslo_concurrency.lockutils [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Lock "3c5781a8-df02-4b4f-ad3f-c04db921748f" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 21.630s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 939.371418] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-663da707-02b4-423c-9de2-7d419a462eaa tempest-ServersAaction247Test-690234416 tempest-ServersAaction247Test-690234416-project-member] Expecting reply to msg 815ef849e436467cae8a781d2b305cdb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 939.382998] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 815ef849e436467cae8a781d2b305cdb [ 939.480160] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52be5ca5-860c-eda9-ad8f-2284f2ea6904, 'name': SearchDatastore_Task, 'duration_secs': 0.009468} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 939.480466] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 939.480750] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 15815ad0-f79a-4031-b625-22c5f93dc2c1/15815ad0-f79a-4031-b625-22c5f93dc2c1.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 939.481037] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-b901ebae-2417-4951-bbab-bd6c02fcdc8d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 939.488408] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 939.488408] env[64020]: value = "task-407945" [ 939.488408] env[64020]: _type = "Task" [ 939.488408] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 939.496051] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407945, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 939.728638] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg 3009eb61e7574658905e3e557040b16b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 939.765679] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3009eb61e7574658905e3e557040b16b [ 939.873304] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 6341a216de0a444b987474c744f30663 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 939.884057] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6341a216de0a444b987474c744f30663 [ 939.998459] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407945, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.433165} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 939.998681] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 15815ad0-f79a-4031-b625-22c5f93dc2c1/15815ad0-f79a-4031-b625-22c5f93dc2c1.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 939.998939] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 939.999246] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-5c55e561-947f-4ab2-8c68-d8d33ccfc446 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 940.005695] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 940.005695] env[64020]: value = "task-407946" [ 940.005695] env[64020]: _type = "Task" [ 940.005695] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 940.013429] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407946, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 940.252780] env[64020]: INFO nova.scheduler.client.report [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Deleted allocations for instance 5554874d-598f-4276-8778-1da6773be649 [ 940.259833] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Expecting reply to msg f04a362d6f8c4b8399a11e74aa20257f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 940.270104] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f04a362d6f8c4b8399a11e74aa20257f [ 940.375751] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 5554874d-598f-4276-8778-1da6773be649 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 940.376126] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 77801ae9-89db-4dc7-af03-0646af73b121 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 940.376170] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 186cd540-6bb2-407f-9adb-9ed841656713 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 940.376266] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 15815ad0-f79a-4031-b625-22c5f93dc2c1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 940.376857] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 68743205a902457ca09423414d2e5490 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 940.386373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 68743205a902457ca09423414d2e5490 [ 940.516596] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407946, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.066014} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 940.516785] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 940.517534] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86cad612-d502-41c5-995f-5d2ca1f0f1cf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 940.537328] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Reconfiguring VM instance instance-00000049 to attach disk [datastore1] 15815ad0-f79a-4031-b625-22c5f93dc2c1/15815ad0-f79a-4031-b625-22c5f93dc2c1.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 940.537590] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-3b8b1a2b-cd9c-4e47-8aac-7282fcc0873c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 940.557790] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 940.557790] env[64020]: value = "task-407947" [ 940.557790] env[64020]: _type = "Task" [ 940.557790] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 940.564594] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407947, 'name': ReconfigVM_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 940.762334] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25139965-af6d-40c7-971b-00974e933552 tempest-ServerRescueNegativeTestJSON-1436166993 tempest-ServerRescueNegativeTestJSON-1436166993-project-member] Lock "5554874d-598f-4276-8778-1da6773be649" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 109.684s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 940.763027] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 48339abee16c461680eb7ddcf811d509 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 940.776246] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 48339abee16c461680eb7ddcf811d509 [ 940.880033] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 51b7673b-15ff-4940-9336-b17bc712f0e3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 940.880033] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg c7bd6e7fa8f54d9496668c7b4793171a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 940.890079] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c7bd6e7fa8f54d9496668c7b4793171a [ 941.068198] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407947, 'name': ReconfigVM_Task} progress is 14%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 941.265838] env[64020]: DEBUG nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 941.268805] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg ded9c86b70f5447b9733bb7ae1cbd334 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 941.310623] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ded9c86b70f5447b9733bb7ae1cbd334 [ 941.382039] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance f2c7916a-aa0d-4b41-9c02-e2cee84acf4e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 941.382711] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg d89eae3d471d48cfbe57b78097fb0436 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 941.393654] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d89eae3d471d48cfbe57b78097fb0436 [ 941.568844] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407947, 'name': ReconfigVM_Task} progress is 14%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 941.794872] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 941.886609] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance a2cd396e-bd03-4c51-8ec6-cd24d7541103 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 941.886609] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 6d62d6c7ebcb446faae016176e3cb7e2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 941.898041] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d62d6c7ebcb446faae016176e3cb7e2 [ 942.070562] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407947, 'name': ReconfigVM_Task, 'duration_secs': 1.112115} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 942.071453] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Reconfigured VM instance instance-00000049 to attach disk [datastore1] 15815ad0-f79a-4031-b625-22c5f93dc2c1/15815ad0-f79a-4031-b625-22c5f93dc2c1.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 942.072610] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-606b8ff1-06a9-4311-8fab-76abc7724aa4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 942.080404] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 942.080404] env[64020]: value = "task-407948" [ 942.080404] env[64020]: _type = "Task" [ 942.080404] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 942.091828] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407948, 'name': Rename_Task} progress is 6%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 942.388966] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 5f485177-eb28-417d-a74c-0e0f30ea7ce2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 942.389982] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 2240d44b9f1545308a05be36fccd5b6f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 942.399997] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2240d44b9f1545308a05be36fccd5b6f [ 942.590549] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407948, 'name': Rename_Task, 'duration_secs': 0.141818} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 942.590959] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 942.591426] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-96d20831-cb6d-43fe-adca-9d3e8da515e5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 942.600320] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 942.600320] env[64020]: value = "task-407949" [ 942.600320] env[64020]: _type = "Task" [ 942.600320] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 942.607396] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407949, 'name': PowerOnVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 942.892480] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance c5e780e2-8ef9-461e-bca2-f9d0039ce3c3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 942.893438] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 72a9c480bb51434da2dacf0ecba1f3a8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 942.912171] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72a9c480bb51434da2dacf0ecba1f3a8 [ 943.108174] env[64020]: DEBUG oslo_vmware.api [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407949, 'name': PowerOnVM_Task, 'duration_secs': 0.404316} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 943.108408] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 943.108600] env[64020]: INFO nova.compute.manager [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Took 5.77 seconds to spawn the instance on the hypervisor. [ 943.108770] env[64020]: DEBUG nova.compute.manager [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 943.109831] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2d0cd4b-0063-4f0d-aa50-bfed90c8d89f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 943.117051] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 43fc977715ef4ce997a207f35588a1e9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 943.152457] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43fc977715ef4ce997a207f35588a1e9 [ 943.397469] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 3ad11bab-4531-486f-88c0-3ef7153ea0f5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 943.397469] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg fcbfe9942944469bb6dfcf08696b21f3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 943.408063] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fcbfe9942944469bb6dfcf08696b21f3 [ 943.626311] env[64020]: INFO nova.compute.manager [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Took 25.53 seconds to build instance. [ 943.627607] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 74f921efe9344071953dd34b0843dd23 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 943.643220] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74f921efe9344071953dd34b0843dd23 [ 943.900329] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 50e9cb9c-10fd-466d-9b11-5175d7955ac9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 943.901165] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 7e369cc08110419ba28651b9b21a50bd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 943.911002] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7e369cc08110419ba28651b9b21a50bd [ 944.106864] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 8e96878d39044468befc1b2a7c6646e5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 944.115346] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8e96878d39044468befc1b2a7c6646e5 [ 944.129211] env[64020]: DEBUG oslo_concurrency.lockutils [None req-016d86f6-aefe-4876-b72b-6cd46f10e261 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "15815ad0-f79a-4031-b625-22c5f93dc2c1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 109.362s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 944.129787] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 36c908dba30042249503e940058b80ec in queue reply_57893177120949e6a93cb88e15cd42b4 [ 944.137875] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36c908dba30042249503e940058b80ec [ 944.404051] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance bf317705-3bf8-4522-b41a-fc023bf766d0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 944.404385] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 73a9e2cb40174298968d81282718ab1c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 944.414001] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 73a9e2cb40174298968d81282718ab1c [ 944.609062] env[64020]: INFO nova.compute.manager [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Rebuilding instance [ 944.631857] env[64020]: DEBUG nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 944.633715] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 36e039e3a12847a99fd5740e1fb695ac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 944.646966] env[64020]: DEBUG nova.compute.manager [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 944.647785] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-959bf924-a4d6-4a50-9eed-3d08b6cbf230 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 944.655445] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg cc7748b00bb64e2ba0243127420d1694 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 944.663940] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36e039e3a12847a99fd5740e1fb695ac [ 944.682417] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cc7748b00bb64e2ba0243127420d1694 [ 944.907155] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 9d199d06-86cb-4d2f-894d-1ce0ef29235f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 944.907730] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 47a1b27e74d64883aa2f80a9fd171bbf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 944.917313] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47a1b27e74d64883aa2f80a9fd171bbf [ 945.150209] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 945.158781] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 945.159137] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-9f24efe2-ca86-402e-9bed-2523b50d6018 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.165794] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 945.165794] env[64020]: value = "task-407950" [ 945.165794] env[64020]: _type = "Task" [ 945.165794] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 945.174117] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407950, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 945.410797] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 3c6ddf08-d0f3-444b-8249-f9dabeeef87f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 945.411128] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Total usable vcpus: 48, total allocated vcpus: 3 {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 945.411128] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1088MB phys_disk=200GB used_disk=3GB total_vcpus=48 used_vcpus=3 pci_stats=[] {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 945.554508] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afb0d9ad-d9f3-49da-b67e-5b952e1ec7dc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.561488] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cdc1bd5-933e-4551-b92b-2645bb2e197c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.589102] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e6c364a-8878-45ce-a6f3-bffa59483f8e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.595310] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-787e4a04-51f5-455c-bc8c-1f1e194dcf11 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.608506] env[64020]: DEBUG nova.compute.provider_tree [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 945.609005] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 9948b9fb467b4c62bfec7f936eb56d46 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 945.616078] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9948b9fb467b4c62bfec7f936eb56d46 [ 945.674629] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407950, 'name': PowerOffVM_Task, 'duration_secs': 0.109983} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 945.674821] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 945.675033] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 945.675727] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d55640e-4b16-4bc6-9e0b-4c700ab6c3b0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.681735] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 945.681936] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-76be49eb-b6e8-4495-aeda-388270647fe0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.704570] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 945.704763] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Deleting contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 945.704933] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Deleting the datastore file [datastore1] 15815ad0-f79a-4031-b625-22c5f93dc2c1 {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 945.705149] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-c46d3e64-12f4-41a4-adbd-744569a37c33 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.710616] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 945.710616] env[64020]: value = "task-407952" [ 945.710616] env[64020]: _type = "Task" [ 945.710616] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 945.717807] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407952, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 946.111723] env[64020]: DEBUG nova.scheduler.client.report [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 946.114097] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 1a687a74d54949148be19754b27a31e8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 946.125642] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a687a74d54949148be19754b27a31e8 [ 946.220533] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407952, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.103294} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 946.220840] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 946.221028] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Deleted contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 946.221240] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 946.222902] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 7c8167e782144781b00c15382dedac89 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 946.251747] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c8167e782144781b00c15382dedac89 [ 946.617153] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=64020) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 946.617514] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 7.772s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 946.617616] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.286s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 946.619686] env[64020]: INFO nova.compute.claims [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 946.621503] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 350cf39f01c844c390aac33f8302437b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 946.654747] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 350cf39f01c844c390aac33f8302437b [ 946.727375] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 84758f262bf34804b545ba478204c1d0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 946.756942] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 84758f262bf34804b545ba478204c1d0 [ 947.125947] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 47aa13e765354ea6abc2711830ee3315 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 947.134136] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47aa13e765354ea6abc2711830ee3315 [ 947.249802] env[64020]: DEBUG nova.virt.hardware [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 947.250059] env[64020]: DEBUG nova.virt.hardware [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 947.250213] env[64020]: DEBUG nova.virt.hardware [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 947.250392] env[64020]: DEBUG nova.virt.hardware [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 947.250535] env[64020]: DEBUG nova.virt.hardware [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 947.250676] env[64020]: DEBUG nova.virt.hardware [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 947.250879] env[64020]: DEBUG nova.virt.hardware [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 947.251033] env[64020]: DEBUG nova.virt.hardware [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 947.251195] env[64020]: DEBUG nova.virt.hardware [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 947.251404] env[64020]: DEBUG nova.virt.hardware [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 947.251575] env[64020]: DEBUG nova.virt.hardware [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 947.252466] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d32a537-2bda-454f-a2c8-2c02c2459e6b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.260582] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d63f04c-9dd6-4c28-8c4d-698cca77478d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.273950] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Instance VIF info [] {{(pid=64020) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 947.279328] env[64020]: DEBUG oslo.service.loopingcall [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 947.279550] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Creating VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 947.279771] env[64020]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-243e2243-d19e-4c3f-8916-1f0ab3ddb77c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.295991] env[64020]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 947.295991] env[64020]: value = "task-407953" [ 947.295991] env[64020]: _type = "Task" [ 947.295991] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 947.304521] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407953, 'name': CreateVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 947.769674] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c42c9c6b-50bb-48f4-bf15-986889010993 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.777153] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a429c162-81d6-40d4-9eb9-e63fa74b4cec {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.808343] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d68a0f0-87b9-4e57-9c94-8a89a57d01b5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.815482] env[64020]: DEBUG oslo_vmware.api [-] Task: {'id': task-407953, 'name': CreateVM_Task, 'duration_secs': 0.238288} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 947.817268] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Created VM on the ESX host {{(pid=64020) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 947.817741] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 947.817899] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 947.818205] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 947.819410] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f138d56-0af8-4805-ab9f-3d5aa9942763 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.822802] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-679ecb53-a7a1-4154-ac74-1bf5ea98f34c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.827107] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 947.827107] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5278b54f-a07f-d0ef-8f2b-d34ead3b6d33" [ 947.827107] env[64020]: _type = "Task" [ 947.827107] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 947.834237] env[64020]: DEBUG nova.compute.provider_tree [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 947.834704] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg b36111c09cfd46a6bdc305014bb894ac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 947.842685] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5278b54f-a07f-d0ef-8f2b-d34ead3b6d33, 'name': SearchDatastore_Task, 'duration_secs': 0.008885} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 947.843172] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b36111c09cfd46a6bdc305014bb894ac [ 947.843589] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 947.843805] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Processing image 9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234 {{(pid=64020) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 947.844038] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 947.844185] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquired lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 947.844358] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 947.844806] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e58799d4-189b-41f9-afa3-3aa0cb1b1008 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.851439] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=64020) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 947.851605] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=64020) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 947.852466] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-301b968b-bd92-433d-bd88-1a74420edf9c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.856803] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 947.856803] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52692bb8-fb08-0483-4d68-8b3e97b1a0e4" [ 947.856803] env[64020]: _type = "Task" [ 947.856803] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 947.863762] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52692bb8-fb08-0483-4d68-8b3e97b1a0e4, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 948.339161] env[64020]: DEBUG nova.scheduler.client.report [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 948.341545] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 2f8a6709391c4d0e8e51bbc89154ee45 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 948.353523] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f8a6709391c4d0e8e51bbc89154ee45 [ 948.367688] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]52692bb8-fb08-0483-4d68-8b3e97b1a0e4, 'name': SearchDatastore_Task, 'duration_secs': 0.007998} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 948.368429] env[64020]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b65a54af-9763-4e25-8b68-cc36258d3759 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 948.373602] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 948.373602] env[64020]: value = "session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5251cd68-892c-840a-0c40-1eb3567d9f7c" [ 948.373602] env[64020]: _type = "Task" [ 948.373602] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 948.380696] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5251cd68-892c-840a-0c40-1eb3567d9f7c, 'name': SearchDatastore_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 948.844379] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.227s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 948.844834] env[64020]: DEBUG nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 948.846748] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg bddc467df357483893fcc98b6ac1a1ce in queue reply_57893177120949e6a93cb88e15cd42b4 [ 948.847831] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.639s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 948.849213] env[64020]: INFO nova.compute.claims [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 948.850823] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 7d2d38e640174500b44d9750bdfc2e84 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 948.878427] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bddc467df357483893fcc98b6ac1a1ce [ 948.884920] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d2d38e640174500b44d9750bdfc2e84 [ 948.885301] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': session[52cf6d64-4b62-459e-1127-9b43ce0f8113]5251cd68-892c-840a-0c40-1eb3567d9f7c, 'name': SearchDatastore_Task, 'duration_secs': 0.008604} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 948.885528] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Releasing lock "[datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 948.885799] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 15815ad0-f79a-4031-b625-22c5f93dc2c1/15815ad0-f79a-4031-b625-22c5f93dc2c1.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 948.886040] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-4e638812-67cf-401f-b8ab-c9835a77dd79 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 948.893337] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 948.893337] env[64020]: value = "task-407954" [ 948.893337] env[64020]: _type = "Task" [ 948.893337] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 948.900904] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407954, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 949.354282] env[64020]: DEBUG nova.compute.utils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 949.354925] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg a68571fea04549548ab3baa78321f285 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 949.356964] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 426baef1f7ca4e159b846f26a3c7368e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 949.357891] env[64020]: DEBUG nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 949.357967] env[64020]: DEBUG nova.network.neutron [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 949.365592] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a68571fea04549548ab3baa78321f285 [ 949.380982] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 426baef1f7ca4e159b846f26a3c7368e [ 949.403089] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407954, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.442498} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 949.403342] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234/9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234.vmdk to [datastore1] 15815ad0-f79a-4031-b625-22c5f93dc2c1/15815ad0-f79a-4031-b625-22c5f93dc2c1.vmdk {{(pid=64020) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 949.403550] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Extending root virtual disk to 1048576 {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 949.403806] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-e9ffe0b9-5f19-4d7f-840f-e290e85d18af {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.410222] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 949.410222] env[64020]: value = "task-407955" [ 949.410222] env[64020]: _type = "Task" [ 949.410222] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 949.418900] env[64020]: DEBUG nova.policy [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9c183cac5f44461daf639a0122e8fee2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6c97516aa6204d9dae64a28689a4d42c', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 949.429001] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407955, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 949.719537] env[64020]: DEBUG nova.network.neutron [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Successfully created port: e3abbc1c-b3d0-47b8-ac47-aff06027311b {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 949.859067] env[64020]: DEBUG nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 949.860985] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 10268cf9627842bfa8db07153d072730 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 949.892069] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10268cf9627842bfa8db07153d072730 [ 949.921407] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407955, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.057843} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 949.921674] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Extended root virtual disk {{(pid=64020) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 949.922481] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bff75203-dcba-44dd-b040-2621ed902f2a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.942462] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Reconfiguring VM instance instance-00000049 to attach disk [datastore1] 15815ad0-f79a-4031-b625-22c5f93dc2c1/15815ad0-f79a-4031-b625-22c5f93dc2c1.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 949.945656] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-9c2f8c24-247f-409f-bf3d-5a0cc055d2c5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.965530] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 949.965530] env[64020]: value = "task-407956" [ 949.965530] env[64020]: _type = "Task" [ 949.965530] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 949.976462] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407956, 'name': ReconfigVM_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 950.069760] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6aca2000-6ea6-4c92-9d5e-b83b5be7f17a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.081108] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-408c093a-f417-4ca8-8fad-be9a17a9c442 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.114098] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-adf7c606-d030-4fd8-8839-6f29307c4b73 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.122103] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1cd7d347-405a-4088-a65b-4ab927dbd9cb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.135097] env[64020]: DEBUG nova.compute.provider_tree [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 950.135591] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg e6cc70ff270047c18e8cf4f9774179a4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 950.142866] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6cc70ff270047c18e8cf4f9774179a4 [ 950.347558] env[64020]: DEBUG nova.compute.manager [req-20bc9728-92ab-4985-b08d-c40eada30f78 req-0647ae65-004d-4f29-99e3-4ffa332d3822 service nova] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Received event network-changed-e3abbc1c-b3d0-47b8-ac47-aff06027311b {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 950.347760] env[64020]: DEBUG nova.compute.manager [req-20bc9728-92ab-4985-b08d-c40eada30f78 req-0647ae65-004d-4f29-99e3-4ffa332d3822 service nova] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Refreshing instance network info cache due to event network-changed-e3abbc1c-b3d0-47b8-ac47-aff06027311b. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 950.348445] env[64020]: DEBUG oslo_concurrency.lockutils [req-20bc9728-92ab-4985-b08d-c40eada30f78 req-0647ae65-004d-4f29-99e3-4ffa332d3822 service nova] Acquiring lock "refresh_cache-51b7673b-15ff-4940-9336-b17bc712f0e3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 950.348598] env[64020]: DEBUG oslo_concurrency.lockutils [req-20bc9728-92ab-4985-b08d-c40eada30f78 req-0647ae65-004d-4f29-99e3-4ffa332d3822 service nova] Acquired lock "refresh_cache-51b7673b-15ff-4940-9336-b17bc712f0e3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 950.348760] env[64020]: DEBUG nova.network.neutron [req-20bc9728-92ab-4985-b08d-c40eada30f78 req-0647ae65-004d-4f29-99e3-4ffa332d3822 service nova] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Refreshing network info cache for port e3abbc1c-b3d0-47b8-ac47-aff06027311b {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 950.349250] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-20bc9728-92ab-4985-b08d-c40eada30f78 req-0647ae65-004d-4f29-99e3-4ffa332d3822 service nova] Expecting reply to msg 865d9c6cd2384f01bcd5de9314c6a7a5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 950.356546] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 865d9c6cd2384f01bcd5de9314c6a7a5 [ 950.367202] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 08ec959921bf473fa1096bb141968f0f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 950.399883] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08ec959921bf473fa1096bb141968f0f [ 950.474663] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407956, 'name': ReconfigVM_Task, 'duration_secs': 0.273221} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 950.474945] env[64020]: DEBUG nova.virt.vmwareapi.volumeops [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Reconfigured VM instance instance-00000049 to attach disk [datastore1] 15815ad0-f79a-4031-b625-22c5f93dc2c1/15815ad0-f79a-4031-b625-22c5f93dc2c1.vmdk or device None with type sparse {{(pid=64020) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 950.475550] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-122f6129-5164-4580-89d5-8f0fc90ec3e3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.482615] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 950.482615] env[64020]: value = "task-407957" [ 950.482615] env[64020]: _type = "Task" [ 950.482615] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 950.489677] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407957, 'name': Rename_Task} progress is 5%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 950.536040] env[64020]: ERROR nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e3abbc1c-b3d0-47b8-ac47-aff06027311b, please check neutron logs for more information. [ 950.536040] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 950.536040] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 950.536040] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 950.536040] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 950.536040] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 950.536040] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 950.536040] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 950.536040] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 950.536040] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 950.536040] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 950.536040] env[64020]: ERROR nova.compute.manager raise self.value [ 950.536040] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 950.536040] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 950.536040] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 950.536040] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 950.536501] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 950.536501] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 950.536501] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e3abbc1c-b3d0-47b8-ac47-aff06027311b, please check neutron logs for more information. [ 950.536501] env[64020]: ERROR nova.compute.manager [ 950.536501] env[64020]: Traceback (most recent call last): [ 950.536501] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 950.536501] env[64020]: listener.cb(fileno) [ 950.536501] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 950.536501] env[64020]: result = function(*args, **kwargs) [ 950.536501] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 950.536501] env[64020]: return func(*args, **kwargs) [ 950.536501] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 950.536501] env[64020]: raise e [ 950.536501] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 950.536501] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 950.536501] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 950.536501] env[64020]: created_port_ids = self._update_ports_for_instance( [ 950.536501] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 950.536501] env[64020]: with excutils.save_and_reraise_exception(): [ 950.536501] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 950.536501] env[64020]: self.force_reraise() [ 950.536501] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 950.536501] env[64020]: raise self.value [ 950.536501] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 950.536501] env[64020]: updated_port = self._update_port( [ 950.536501] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 950.536501] env[64020]: _ensure_no_port_binding_failure(port) [ 950.536501] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 950.536501] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 950.537485] env[64020]: nova.exception.PortBindingFailed: Binding failed for port e3abbc1c-b3d0-47b8-ac47-aff06027311b, please check neutron logs for more information. [ 950.537485] env[64020]: Removing descriptor: 18 [ 950.638085] env[64020]: DEBUG nova.scheduler.client.report [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 950.640801] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 7f9e87f0963843c1998f4d4d6c2c1602 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 950.652035] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f9e87f0963843c1998f4d4d6c2c1602 [ 950.867655] env[64020]: DEBUG nova.network.neutron [req-20bc9728-92ab-4985-b08d-c40eada30f78 req-0647ae65-004d-4f29-99e3-4ffa332d3822 service nova] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 950.870047] env[64020]: DEBUG nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 950.893795] env[64020]: DEBUG nova.virt.hardware [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 950.894031] env[64020]: DEBUG nova.virt.hardware [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 950.894183] env[64020]: DEBUG nova.virt.hardware [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 950.894360] env[64020]: DEBUG nova.virt.hardware [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 950.894504] env[64020]: DEBUG nova.virt.hardware [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 950.894648] env[64020]: DEBUG nova.virt.hardware [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 950.894849] env[64020]: DEBUG nova.virt.hardware [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 950.895000] env[64020]: DEBUG nova.virt.hardware [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 950.895161] env[64020]: DEBUG nova.virt.hardware [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 950.895320] env[64020]: DEBUG nova.virt.hardware [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 950.895487] env[64020]: DEBUG nova.virt.hardware [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 950.896330] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f8d3a72-5935-46d3-8b1c-0dcb8759a9a7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.904088] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28ee3f96-39b8-41e0-9c84-34591ba7ef62 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.919564] env[64020]: ERROR nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e3abbc1c-b3d0-47b8-ac47-aff06027311b, please check neutron logs for more information. [ 950.919564] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Traceback (most recent call last): [ 950.919564] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 950.919564] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] yield resources [ 950.919564] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 950.919564] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] self.driver.spawn(context, instance, image_meta, [ 950.919564] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 950.919564] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 950.919564] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 950.919564] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] vm_ref = self.build_virtual_machine(instance, [ 950.919564] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] vif_infos = vmwarevif.get_vif_info(self._session, [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] for vif in network_info: [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] return self._sync_wrapper(fn, *args, **kwargs) [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] self.wait() [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] self[:] = self._gt.wait() [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] return self._exit_event.wait() [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 950.919982] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] current.throw(*self._exc) [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] result = function(*args, **kwargs) [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] return func(*args, **kwargs) [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] raise e [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] nwinfo = self.network_api.allocate_for_instance( [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] created_port_ids = self._update_ports_for_instance( [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] with excutils.save_and_reraise_exception(): [ 950.920397] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 950.920885] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] self.force_reraise() [ 950.920885] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 950.920885] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] raise self.value [ 950.920885] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 950.920885] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] updated_port = self._update_port( [ 950.920885] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 950.920885] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] _ensure_no_port_binding_failure(port) [ 950.920885] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 950.920885] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] raise exception.PortBindingFailed(port_id=port['id']) [ 950.920885] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] nova.exception.PortBindingFailed: Binding failed for port e3abbc1c-b3d0-47b8-ac47-aff06027311b, please check neutron logs for more information. [ 950.920885] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] [ 950.920885] env[64020]: INFO nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Terminating instance [ 950.921827] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquiring lock "refresh_cache-51b7673b-15ff-4940-9336-b17bc712f0e3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 950.944746] env[64020]: DEBUG nova.network.neutron [req-20bc9728-92ab-4985-b08d-c40eada30f78 req-0647ae65-004d-4f29-99e3-4ffa332d3822 service nova] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 950.945196] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-20bc9728-92ab-4985-b08d-c40eada30f78 req-0647ae65-004d-4f29-99e3-4ffa332d3822 service nova] Expecting reply to msg 315a4cc00ed149258a2025406a2a2278 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 950.953419] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 315a4cc00ed149258a2025406a2a2278 [ 950.991543] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407957, 'name': Rename_Task, 'duration_secs': 0.12882} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 950.991788] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Powering on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 950.992067] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-a561f686-ebbe-4bc2-96cf-18545e3d886a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.997803] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 950.997803] env[64020]: value = "task-407958" [ 950.997803] env[64020]: _type = "Task" [ 950.997803] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 951.005972] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407958, 'name': PowerOnVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 951.144154] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.296s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 951.144694] env[64020]: DEBUG nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 951.146442] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg d57d961a42f5443e8888f5252a5cf1c0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 951.147533] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 25.938s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 951.149094] env[64020]: INFO nova.compute.claims [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 951.150725] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 4a83d1ac66264ac48342c8f2f77c6fa2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 951.181805] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d57d961a42f5443e8888f5252a5cf1c0 [ 951.183298] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4a83d1ac66264ac48342c8f2f77c6fa2 [ 951.447111] env[64020]: DEBUG oslo_concurrency.lockutils [req-20bc9728-92ab-4985-b08d-c40eada30f78 req-0647ae65-004d-4f29-99e3-4ffa332d3822 service nova] Releasing lock "refresh_cache-51b7673b-15ff-4940-9336-b17bc712f0e3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 951.447514] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquired lock "refresh_cache-51b7673b-15ff-4940-9336-b17bc712f0e3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 951.447700] env[64020]: DEBUG nova.network.neutron [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 951.448154] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 2225e98f2ce142f58aebb00eeb6c4f3c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 951.454675] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2225e98f2ce142f58aebb00eeb6c4f3c [ 951.507516] env[64020]: DEBUG oslo_vmware.api [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407958, 'name': PowerOnVM_Task, 'duration_secs': 0.414101} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 951.507772] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Powered on the VM {{(pid=64020) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 951.507966] env[64020]: DEBUG nova.compute.manager [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Checking state {{(pid=64020) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 951.508703] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8f8130d-b461-4722-b854-6ca1d250af6e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.515712] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 2317a0c24caf41f7aa927f5b3b55fa95 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 951.546086] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2317a0c24caf41f7aa927f5b3b55fa95 [ 951.653564] env[64020]: DEBUG nova.compute.utils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 951.654209] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 8cf8ddacaf6d4b8e985cf5e421d6649f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 951.656252] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 8311e5b4aeba4933a2ab70e664ba6751 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 951.657228] env[64020]: DEBUG nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 951.657396] env[64020]: DEBUG nova.network.neutron [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 951.664431] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8cf8ddacaf6d4b8e985cf5e421d6649f [ 951.665875] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8311e5b4aeba4933a2ab70e664ba6751 [ 951.717459] env[64020]: DEBUG nova.policy [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd496b47ef9db47fa97323baf6bbd63ed', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f1fb1191ee044c4f921f5b7935f0109d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 951.963040] env[64020]: DEBUG nova.network.neutron [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Successfully created port: d8633a09-a42e-4059-b375-94f01ce09ebc {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 951.965513] env[64020]: DEBUG nova.network.neutron [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 952.025918] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 952.158058] env[64020]: DEBUG nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 952.159846] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 1b2e120b725747f9b102aafa0cdfe6f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 952.192668] env[64020]: DEBUG nova.network.neutron [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 952.193276] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 029fb939fdd5427aa38f1bb0b8444745 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 952.201738] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b2e120b725747f9b102aafa0cdfe6f5 [ 952.230371] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 029fb939fdd5427aa38f1bb0b8444745 [ 952.353603] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c7b9f18-3e05-474a-a716-115794b3c88a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.364493] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6e28917-4141-4392-96f3-db0b53167bcb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.397534] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8536de2d-093a-4be5-9c9f-a7cc24957996 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.404040] env[64020]: DEBUG nova.compute.manager [req-6c3b6374-f514-4cba-948b-6cc2711ce6c6 req-72ae49b4-88f0-4d4c-b30e-1a0f34b87c94 service nova] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Received event network-vif-deleted-e3abbc1c-b3d0-47b8-ac47-aff06027311b {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 952.407978] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2bb4aeb-024a-4cae-bacb-6eddd70697d8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.424098] env[64020]: DEBUG nova.compute.provider_tree [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 952.424674] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg b80593336d354cc69d3fa5686c80247f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 952.431999] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b80593336d354cc69d3fa5686c80247f [ 952.667910] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 7e0fd5ee7f5d4b47bdf4dd4f0eb86dfc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 952.698324] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Releasing lock "refresh_cache-51b7673b-15ff-4940-9336-b17bc712f0e3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 952.698324] env[64020]: DEBUG nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 952.698324] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 952.698324] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-57db12d5-84de-49b2-ad74-df2b0097a105 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.705459] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8eac3d0b-3fdf-430d-a855-b44117e8fbb1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.717713] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7e0fd5ee7f5d4b47bdf4dd4f0eb86dfc [ 952.732269] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 51b7673b-15ff-4940-9336-b17bc712f0e3 could not be found. [ 952.732820] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 952.733123] env[64020]: INFO nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Took 0.04 seconds to destroy the instance on the hypervisor. [ 952.733576] env[64020]: DEBUG oslo.service.loopingcall [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 952.733915] env[64020]: DEBUG nova.compute.manager [-] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 952.734125] env[64020]: DEBUG nova.network.neutron [-] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 952.748620] env[64020]: DEBUG nova.network.neutron [-] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 952.749336] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0fd4f1285d764f3b82219741660b9de4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 952.758310] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0fd4f1285d764f3b82219741660b9de4 [ 952.897022] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 844f72b10b0a4e55876ba6c48155205e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 952.908196] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 844f72b10b0a4e55876ba6c48155205e [ 952.925380] env[64020]: ERROR nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d8633a09-a42e-4059-b375-94f01ce09ebc, please check neutron logs for more information. [ 952.925380] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 952.925380] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 952.925380] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 952.925380] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 952.925380] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 952.925380] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 952.925380] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 952.925380] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 952.925380] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 952.925380] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 952.925380] env[64020]: ERROR nova.compute.manager raise self.value [ 952.925380] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 952.925380] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 952.925380] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 952.925380] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 952.925866] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 952.925866] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 952.925866] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d8633a09-a42e-4059-b375-94f01ce09ebc, please check neutron logs for more information. [ 952.925866] env[64020]: ERROR nova.compute.manager [ 952.925866] env[64020]: Traceback (most recent call last): [ 952.925866] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 952.925866] env[64020]: listener.cb(fileno) [ 952.925866] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 952.925866] env[64020]: result = function(*args, **kwargs) [ 952.925866] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 952.925866] env[64020]: return func(*args, **kwargs) [ 952.925866] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 952.925866] env[64020]: raise e [ 952.925866] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 952.925866] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 952.925866] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 952.925866] env[64020]: created_port_ids = self._update_ports_for_instance( [ 952.925866] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 952.925866] env[64020]: with excutils.save_and_reraise_exception(): [ 952.925866] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 952.925866] env[64020]: self.force_reraise() [ 952.925866] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 952.925866] env[64020]: raise self.value [ 952.925866] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 952.925866] env[64020]: updated_port = self._update_port( [ 952.925866] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 952.925866] env[64020]: _ensure_no_port_binding_failure(port) [ 952.925866] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 952.925866] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 952.926764] env[64020]: nova.exception.PortBindingFailed: Binding failed for port d8633a09-a42e-4059-b375-94f01ce09ebc, please check neutron logs for more information. [ 952.926764] env[64020]: Removing descriptor: 18 [ 952.926764] env[64020]: DEBUG nova.scheduler.client.report [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 952.929027] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg d40c22cf8d6f4b15a5850007ac1e62db in queue reply_57893177120949e6a93cb88e15cd42b4 [ 952.943734] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d40c22cf8d6f4b15a5850007ac1e62db [ 953.170070] env[64020]: DEBUG nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 953.194076] env[64020]: DEBUG nova.virt.hardware [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 953.194305] env[64020]: DEBUG nova.virt.hardware [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 953.194458] env[64020]: DEBUG nova.virt.hardware [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 953.194634] env[64020]: DEBUG nova.virt.hardware [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 953.194778] env[64020]: DEBUG nova.virt.hardware [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 953.194988] env[64020]: DEBUG nova.virt.hardware [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 953.195201] env[64020]: DEBUG nova.virt.hardware [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 953.195357] env[64020]: DEBUG nova.virt.hardware [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 953.195516] env[64020]: DEBUG nova.virt.hardware [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 953.195672] env[64020]: DEBUG nova.virt.hardware [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 953.195839] env[64020]: DEBUG nova.virt.hardware [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 953.196950] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c406094-90d6-4d84-8024-72e63c5412ee {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.205334] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1e050fa-c339-4459-8367-b97debad08bf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.218128] env[64020]: ERROR nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d8633a09-a42e-4059-b375-94f01ce09ebc, please check neutron logs for more information. [ 953.218128] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Traceback (most recent call last): [ 953.218128] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 953.218128] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] yield resources [ 953.218128] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 953.218128] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] self.driver.spawn(context, instance, image_meta, [ 953.218128] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 953.218128] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 953.218128] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 953.218128] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] vm_ref = self.build_virtual_machine(instance, [ 953.218128] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] vif_infos = vmwarevif.get_vif_info(self._session, [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] for vif in network_info: [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] return self._sync_wrapper(fn, *args, **kwargs) [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] self.wait() [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] self[:] = self._gt.wait() [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] return self._exit_event.wait() [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 953.218528] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] current.throw(*self._exc) [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] result = function(*args, **kwargs) [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] return func(*args, **kwargs) [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] raise e [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] nwinfo = self.network_api.allocate_for_instance( [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] created_port_ids = self._update_ports_for_instance( [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] with excutils.save_and_reraise_exception(): [ 953.218933] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 953.219330] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] self.force_reraise() [ 953.219330] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 953.219330] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] raise self.value [ 953.219330] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 953.219330] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] updated_port = self._update_port( [ 953.219330] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 953.219330] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] _ensure_no_port_binding_failure(port) [ 953.219330] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 953.219330] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] raise exception.PortBindingFailed(port_id=port['id']) [ 953.219330] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] nova.exception.PortBindingFailed: Binding failed for port d8633a09-a42e-4059-b375-94f01ce09ebc, please check neutron logs for more information. [ 953.219330] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] [ 953.219330] env[64020]: INFO nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Terminating instance [ 953.220272] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 953.220428] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 953.220586] env[64020]: DEBUG nova.network.neutron [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 953.220973] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg dc0ee2ab0ceb427aa9fb7069126b1b46 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 953.227887] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc0ee2ab0ceb427aa9fb7069126b1b46 [ 953.251504] env[64020]: DEBUG nova.network.neutron [-] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 953.251952] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg fe1e1ced601d413d9679179a40cf3e79 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 953.259747] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe1e1ced601d413d9679179a40cf3e79 [ 953.399737] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "15815ad0-f79a-4031-b625-22c5f93dc2c1" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 953.399982] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "15815ad0-f79a-4031-b625-22c5f93dc2c1" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 953.400210] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "15815ad0-f79a-4031-b625-22c5f93dc2c1-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 953.400384] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "15815ad0-f79a-4031-b625-22c5f93dc2c1-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 953.400568] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "15815ad0-f79a-4031-b625-22c5f93dc2c1-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 953.402708] env[64020]: INFO nova.compute.manager [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Terminating instance [ 953.404320] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "refresh_cache-15815ad0-f79a-4031-b625-22c5f93dc2c1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 953.404475] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquired lock "refresh_cache-15815ad0-f79a-4031-b625-22c5f93dc2c1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 953.404636] env[64020]: DEBUG nova.network.neutron [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 953.405024] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 6a2283e3f8a94c9fa94b336c09a25b0f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 953.411481] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a2283e3f8a94c9fa94b336c09a25b0f [ 953.430809] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.283s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 953.431407] env[64020]: DEBUG nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 953.433742] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 312dc2e189f9475a856b3da1f79bd458 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 953.435020] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.347s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 953.436433] env[64020]: INFO nova.compute.claims [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 953.438296] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 4827ffe62c2841ecaf73e105d883115b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 953.469244] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 312dc2e189f9475a856b3da1f79bd458 [ 953.471708] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4827ffe62c2841ecaf73e105d883115b [ 953.739627] env[64020]: DEBUG nova.network.neutron [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 953.754144] env[64020]: INFO nova.compute.manager [-] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Took 1.02 seconds to deallocate network for instance. [ 953.756861] env[64020]: DEBUG nova.compute.claims [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 953.757061] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 953.831484] env[64020]: DEBUG nova.network.neutron [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 953.832340] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg d62cf243826542e99fec40671981459d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 953.840837] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d62cf243826542e99fec40671981459d [ 953.923835] env[64020]: DEBUG nova.network.neutron [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 953.940965] env[64020]: DEBUG nova.compute.utils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 953.941718] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg ef5f5b7d09be4399a0e8835f7f99e423 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 953.943681] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 19225947427744c49a99dc74b5a83de9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 953.944704] env[64020]: DEBUG nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 953.944863] env[64020]: DEBUG nova.network.neutron [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 953.952257] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef5f5b7d09be4399a0e8835f7f99e423 [ 953.953765] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19225947427744c49a99dc74b5a83de9 [ 953.981714] env[64020]: DEBUG nova.network.neutron [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 953.982244] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 4e2195a6735a448683509d77947210b7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 953.984677] env[64020]: DEBUG nova.policy [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02b993f5bbd5442a8d2c06cff095e867', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e0c4cb7c0e9941f7a056fef708be822c', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 953.990114] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e2195a6735a448683509d77947210b7 [ 954.254245] env[64020]: DEBUG nova.network.neutron [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Successfully created port: 3df624d0-05e9-4386-b3b2-b6915a730dcb {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 954.334969] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 954.335415] env[64020]: DEBUG nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 954.335601] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 954.335906] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-12544e70-4279-4c7b-a308-d9946bf2652e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.345469] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f876f06-386f-4703-9017-0ae7ded9e7fd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.368295] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f2c7916a-aa0d-4b41-9c02-e2cee84acf4e could not be found. [ 954.368533] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 954.368710] env[64020]: INFO nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Took 0.03 seconds to destroy the instance on the hypervisor. [ 954.368948] env[64020]: DEBUG oslo.service.loopingcall [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 954.369172] env[64020]: DEBUG nova.compute.manager [-] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 954.369278] env[64020]: DEBUG nova.network.neutron [-] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 954.385851] env[64020]: DEBUG nova.network.neutron [-] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 954.386320] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b3d49321cbdb480b948fe73f53e04949 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 954.394182] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3d49321cbdb480b948fe73f53e04949 [ 954.430600] env[64020]: DEBUG nova.compute.manager [req-4b2a77c8-0f87-4ad0-a089-8e47555e2deb req-9d032231-0bce-4233-94e7-4545341df77b service nova] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Received event network-changed-d8633a09-a42e-4059-b375-94f01ce09ebc {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 954.430812] env[64020]: DEBUG nova.compute.manager [req-4b2a77c8-0f87-4ad0-a089-8e47555e2deb req-9d032231-0bce-4233-94e7-4545341df77b service nova] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Refreshing instance network info cache due to event network-changed-d8633a09-a42e-4059-b375-94f01ce09ebc. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 954.430998] env[64020]: DEBUG oslo_concurrency.lockutils [req-4b2a77c8-0f87-4ad0-a089-8e47555e2deb req-9d032231-0bce-4233-94e7-4545341df77b service nova] Acquiring lock "refresh_cache-f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 954.431135] env[64020]: DEBUG oslo_concurrency.lockutils [req-4b2a77c8-0f87-4ad0-a089-8e47555e2deb req-9d032231-0bce-4233-94e7-4545341df77b service nova] Acquired lock "refresh_cache-f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 954.431314] env[64020]: DEBUG nova.network.neutron [req-4b2a77c8-0f87-4ad0-a089-8e47555e2deb req-9d032231-0bce-4233-94e7-4545341df77b service nova] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Refreshing network info cache for port d8633a09-a42e-4059-b375-94f01ce09ebc {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 954.431737] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-4b2a77c8-0f87-4ad0-a089-8e47555e2deb req-9d032231-0bce-4233-94e7-4545341df77b service nova] Expecting reply to msg 5e08e656d9034f5e956a26c20d84a87a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 954.438747] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e08e656d9034f5e956a26c20d84a87a [ 954.445110] env[64020]: DEBUG nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 954.446843] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg db0c3198a6f643a1a3586378ec7b0251 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 954.483099] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db0c3198a6f643a1a3586378ec7b0251 [ 954.483817] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Releasing lock "refresh_cache-15815ad0-f79a-4031-b625-22c5f93dc2c1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 954.484233] env[64020]: DEBUG nova.compute.manager [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 954.484417] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 954.485567] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89743f7f-18e9-423c-8326-a216ff656cbe {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.496728] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 954.499197] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-750c7993-ed30-4934-a39c-25fa5b69a96c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.506418] env[64020]: DEBUG oslo_vmware.api [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 954.506418] env[64020]: value = "task-407959" [ 954.506418] env[64020]: _type = "Task" [ 954.506418] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 954.517102] env[64020]: DEBUG oslo_vmware.api [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407959, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 954.641561] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b114c70-154f-41cb-9b5f-97e0a7b429fb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.649186] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-100eab60-364b-42c3-a439-67b7c6edd460 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.679264] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e52434d8-67a7-48f2-9054-60b7d558603a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.686548] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27cea9a1-1603-4c23-9f70-c7ca63b147d2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.701359] env[64020]: DEBUG nova.compute.provider_tree [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 954.701885] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg c5433a33fe6447ac9bfb42ee5c5c88f1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 954.710381] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5433a33fe6447ac9bfb42ee5c5c88f1 [ 954.894763] env[64020]: DEBUG nova.network.neutron [-] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 954.895254] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg cd921166f7354a15ac5b3e5823073eda in queue reply_57893177120949e6a93cb88e15cd42b4 [ 954.905044] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd921166f7354a15ac5b3e5823073eda [ 954.954646] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 60b622b4bf5546d1af03973d9f74c849 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 954.956292] env[64020]: DEBUG nova.network.neutron [req-4b2a77c8-0f87-4ad0-a089-8e47555e2deb req-9d032231-0bce-4233-94e7-4545341df77b service nova] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 954.985649] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60b622b4bf5546d1af03973d9f74c849 [ 955.015720] env[64020]: DEBUG oslo_vmware.api [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407959, 'name': PowerOffVM_Task, 'duration_secs': 0.175142} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 955.015982] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 955.016156] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 955.016393] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-91160c5d-b9e7-4439-8f97-f757b7c234da {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 955.032915] env[64020]: DEBUG nova.network.neutron [req-4b2a77c8-0f87-4ad0-a089-8e47555e2deb req-9d032231-0bce-4233-94e7-4545341df77b service nova] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 955.033443] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-4b2a77c8-0f87-4ad0-a089-8e47555e2deb req-9d032231-0bce-4233-94e7-4545341df77b service nova] Expecting reply to msg 4ff292a4b3f0411494c43a0d139589e7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 955.038934] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 955.039165] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Deleting contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 955.039358] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Deleting the datastore file [datastore1] 15815ad0-f79a-4031-b625-22c5f93dc2c1 {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 955.039758] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-89ae0064-04ed-4ff0-a507-a61d1e714595 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 955.042557] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4ff292a4b3f0411494c43a0d139589e7 [ 955.046356] env[64020]: DEBUG oslo_vmware.api [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 955.046356] env[64020]: value = "task-407961" [ 955.046356] env[64020]: _type = "Task" [ 955.046356] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 955.060103] env[64020]: DEBUG oslo_vmware.api [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407961, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 955.081027] env[64020]: DEBUG nova.compute.manager [req-5045e268-37a8-4935-861f-fc30175fe6f6 req-aa18ca20-b7a1-488c-bc5b-29ebe704af2f service nova] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Received event network-changed-3df624d0-05e9-4386-b3b2-b6915a730dcb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 955.081223] env[64020]: DEBUG nova.compute.manager [req-5045e268-37a8-4935-861f-fc30175fe6f6 req-aa18ca20-b7a1-488c-bc5b-29ebe704af2f service nova] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Refreshing instance network info cache due to event network-changed-3df624d0-05e9-4386-b3b2-b6915a730dcb. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 955.081445] env[64020]: DEBUG oslo_concurrency.lockutils [req-5045e268-37a8-4935-861f-fc30175fe6f6 req-aa18ca20-b7a1-488c-bc5b-29ebe704af2f service nova] Acquiring lock "refresh_cache-a2cd396e-bd03-4c51-8ec6-cd24d7541103" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 955.081580] env[64020]: DEBUG oslo_concurrency.lockutils [req-5045e268-37a8-4935-861f-fc30175fe6f6 req-aa18ca20-b7a1-488c-bc5b-29ebe704af2f service nova] Acquired lock "refresh_cache-a2cd396e-bd03-4c51-8ec6-cd24d7541103" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 955.081730] env[64020]: DEBUG nova.network.neutron [req-5045e268-37a8-4935-861f-fc30175fe6f6 req-aa18ca20-b7a1-488c-bc5b-29ebe704af2f service nova] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Refreshing network info cache for port 3df624d0-05e9-4386-b3b2-b6915a730dcb {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 955.082238] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-5045e268-37a8-4935-861f-fc30175fe6f6 req-aa18ca20-b7a1-488c-bc5b-29ebe704af2f service nova] Expecting reply to msg 4c0ab9ccc7cf43b0af4e5affe3ea2d89 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 955.088601] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c0ab9ccc7cf43b0af4e5affe3ea2d89 [ 955.204823] env[64020]: DEBUG nova.scheduler.client.report [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 955.207142] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 4d381b0597ee457fb9335b8fc474894e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 955.217541] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d381b0597ee457fb9335b8fc474894e [ 955.236610] env[64020]: ERROR nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3df624d0-05e9-4386-b3b2-b6915a730dcb, please check neutron logs for more information. [ 955.236610] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 955.236610] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 955.236610] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 955.236610] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 955.236610] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 955.236610] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 955.236610] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 955.236610] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 955.236610] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 955.236610] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 955.236610] env[64020]: ERROR nova.compute.manager raise self.value [ 955.236610] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 955.236610] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 955.236610] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 955.236610] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 955.237091] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 955.237091] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 955.237091] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3df624d0-05e9-4386-b3b2-b6915a730dcb, please check neutron logs for more information. [ 955.237091] env[64020]: ERROR nova.compute.manager [ 955.237091] env[64020]: Traceback (most recent call last): [ 955.237091] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 955.237091] env[64020]: listener.cb(fileno) [ 955.237091] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 955.237091] env[64020]: result = function(*args, **kwargs) [ 955.237091] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 955.237091] env[64020]: return func(*args, **kwargs) [ 955.237091] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 955.237091] env[64020]: raise e [ 955.237091] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 955.237091] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 955.237091] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 955.237091] env[64020]: created_port_ids = self._update_ports_for_instance( [ 955.237091] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 955.237091] env[64020]: with excutils.save_and_reraise_exception(): [ 955.237091] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 955.237091] env[64020]: self.force_reraise() [ 955.237091] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 955.237091] env[64020]: raise self.value [ 955.237091] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 955.237091] env[64020]: updated_port = self._update_port( [ 955.237091] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 955.237091] env[64020]: _ensure_no_port_binding_failure(port) [ 955.237091] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 955.237091] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 955.237956] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 3df624d0-05e9-4386-b3b2-b6915a730dcb, please check neutron logs for more information. [ 955.237956] env[64020]: Removing descriptor: 16 [ 955.401845] env[64020]: INFO nova.compute.manager [-] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Took 1.03 seconds to deallocate network for instance. [ 955.404164] env[64020]: DEBUG nova.compute.claims [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 955.404342] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 955.459972] env[64020]: DEBUG nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 955.493479] env[64020]: DEBUG nova.virt.hardware [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 955.493745] env[64020]: DEBUG nova.virt.hardware [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 955.493904] env[64020]: DEBUG nova.virt.hardware [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 955.494084] env[64020]: DEBUG nova.virt.hardware [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 955.494229] env[64020]: DEBUG nova.virt.hardware [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 955.494373] env[64020]: DEBUG nova.virt.hardware [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 955.494576] env[64020]: DEBUG nova.virt.hardware [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 955.494772] env[64020]: DEBUG nova.virt.hardware [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 955.494891] env[64020]: DEBUG nova.virt.hardware [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 955.495047] env[64020]: DEBUG nova.virt.hardware [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 955.495268] env[64020]: DEBUG nova.virt.hardware [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 955.496109] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21e2a4b7-02e8-4d52-a7c4-438aa9b1fde4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 955.504517] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd0b94b8-88f0-4bec-923a-e48ec89e4010 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 955.517772] env[64020]: ERROR nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3df624d0-05e9-4386-b3b2-b6915a730dcb, please check neutron logs for more information. [ 955.517772] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Traceback (most recent call last): [ 955.517772] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 955.517772] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] yield resources [ 955.517772] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 955.517772] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] self.driver.spawn(context, instance, image_meta, [ 955.517772] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 955.517772] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] self._vmops.spawn(context, instance, image_meta, injected_files, [ 955.517772] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 955.517772] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] vm_ref = self.build_virtual_machine(instance, [ 955.517772] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] vif_infos = vmwarevif.get_vif_info(self._session, [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] for vif in network_info: [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] return self._sync_wrapper(fn, *args, **kwargs) [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] self.wait() [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] self[:] = self._gt.wait() [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] return self._exit_event.wait() [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 955.518122] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] current.throw(*self._exc) [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] result = function(*args, **kwargs) [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] return func(*args, **kwargs) [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] raise e [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] nwinfo = self.network_api.allocate_for_instance( [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] created_port_ids = self._update_ports_for_instance( [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] with excutils.save_and_reraise_exception(): [ 955.518598] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 955.518985] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] self.force_reraise() [ 955.518985] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 955.518985] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] raise self.value [ 955.518985] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 955.518985] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] updated_port = self._update_port( [ 955.518985] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 955.518985] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] _ensure_no_port_binding_failure(port) [ 955.518985] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 955.518985] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] raise exception.PortBindingFailed(port_id=port['id']) [ 955.518985] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] nova.exception.PortBindingFailed: Binding failed for port 3df624d0-05e9-4386-b3b2-b6915a730dcb, please check neutron logs for more information. [ 955.518985] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] [ 955.518985] env[64020]: INFO nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Terminating instance [ 955.520239] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-a2cd396e-bd03-4c51-8ec6-cd24d7541103" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 955.535755] env[64020]: DEBUG oslo_concurrency.lockutils [req-4b2a77c8-0f87-4ad0-a089-8e47555e2deb req-9d032231-0bce-4233-94e7-4545341df77b service nova] Releasing lock "refresh_cache-f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 955.535971] env[64020]: DEBUG nova.compute.manager [req-4b2a77c8-0f87-4ad0-a089-8e47555e2deb req-9d032231-0bce-4233-94e7-4545341df77b service nova] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Received event network-vif-deleted-d8633a09-a42e-4059-b375-94f01ce09ebc {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 955.557241] env[64020]: DEBUG oslo_vmware.api [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407961, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.129204} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 955.557475] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 955.557648] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Deleted contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 955.557814] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 955.557975] env[64020]: INFO nova.compute.manager [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Took 1.07 seconds to destroy the instance on the hypervisor. [ 955.558196] env[64020]: DEBUG oslo.service.loopingcall [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 955.558375] env[64020]: DEBUG nova.compute.manager [-] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 955.558462] env[64020]: DEBUG nova.network.neutron [-] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 955.572144] env[64020]: DEBUG nova.network.neutron [-] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 955.572680] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg d7fbc431c985442ab7ec38d8063fbe1b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 955.578746] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7fbc431c985442ab7ec38d8063fbe1b [ 955.596793] env[64020]: DEBUG nova.network.neutron [req-5045e268-37a8-4935-861f-fc30175fe6f6 req-aa18ca20-b7a1-488c-bc5b-29ebe704af2f service nova] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 955.661480] env[64020]: DEBUG nova.network.neutron [req-5045e268-37a8-4935-861f-fc30175fe6f6 req-aa18ca20-b7a1-488c-bc5b-29ebe704af2f service nova] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 955.662016] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-5045e268-37a8-4935-861f-fc30175fe6f6 req-aa18ca20-b7a1-488c-bc5b-29ebe704af2f service nova] Expecting reply to msg c6ed368239744b36b0ac7e16fb7c8afa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 955.669756] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c6ed368239744b36b0ac7e16fb7c8afa [ 955.709293] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.274s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 955.709799] env[64020]: DEBUG nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 955.711481] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 844842d251d44c529a04ecd78899c9cb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 955.712743] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.743s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 955.714111] env[64020]: INFO nova.compute.claims [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 955.715562] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg d3812b1eaaae4d4d8a7acf683083f3c4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 955.747813] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 844842d251d44c529a04ecd78899c9cb [ 955.749854] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d3812b1eaaae4d4d8a7acf683083f3c4 [ 956.074342] env[64020]: DEBUG nova.network.neutron [-] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 956.074822] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 52870e3bf910471188c4eb53d996923e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 956.083249] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52870e3bf910471188c4eb53d996923e [ 956.164462] env[64020]: DEBUG oslo_concurrency.lockutils [req-5045e268-37a8-4935-861f-fc30175fe6f6 req-aa18ca20-b7a1-488c-bc5b-29ebe704af2f service nova] Releasing lock "refresh_cache-a2cd396e-bd03-4c51-8ec6-cd24d7541103" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 956.164866] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-a2cd396e-bd03-4c51-8ec6-cd24d7541103" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 956.165066] env[64020]: DEBUG nova.network.neutron [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 956.165448] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 2713f37e365e45109d3a012f0a3a3336 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 956.180808] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2713f37e365e45109d3a012f0a3a3336 [ 956.217941] env[64020]: DEBUG nova.compute.utils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 956.218571] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 14f4e38f7ba1433199bfffa774f48aba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 956.220502] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg e3ed6288d1824f7c834d1ed9c1f5efb6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 956.221421] env[64020]: DEBUG nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 956.221587] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 956.227171] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3ed6288d1824f7c834d1ed9c1f5efb6 [ 956.228383] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14f4e38f7ba1433199bfffa774f48aba [ 956.276472] env[64020]: DEBUG nova.policy [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '17c497749a024ed389ed988afd37327c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '74dfe9d51d8e4237b10602ad703c21f6', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 956.539370] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Successfully created port: 3e907ae1-99ed-4439-863b-3142d5834117 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 956.577505] env[64020]: INFO nova.compute.manager [-] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Took 1.02 seconds to deallocate network for instance. [ 956.581384] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg e6946706b1ca4863ae10a4548a9ef851 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 956.609148] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6946706b1ca4863ae10a4548a9ef851 [ 956.682940] env[64020]: DEBUG nova.network.neutron [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 956.722114] env[64020]: DEBUG nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 956.723924] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg e587ee658162489ca274eb3c5a4c9081 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 956.760205] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e587ee658162489ca274eb3c5a4c9081 [ 956.776924] env[64020]: DEBUG nova.network.neutron [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 956.777422] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 1a7395e376084ff69ba22b32826eb4b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 956.785553] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a7395e376084ff69ba22b32826eb4b3 [ 956.801830] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Successfully created port: b2283bfe-9873-4f6e-ae9f-61c4a880251b {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 956.913947] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c12c6724-36df-46db-aa5b-249b3f02d582 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 956.922639] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a832a921-5f7a-45ac-a7d6-b24cf00c6142 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 956.953697] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57c08e95-75a5-4ea2-9fd6-1dddc63a5331 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 956.961329] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-346c4771-25f9-425e-b862-d6f10aa3eff1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 956.975100] env[64020]: DEBUG nova.compute.provider_tree [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 956.975611] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 504206f9332d4aeca098ed2a7c5aebd6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 956.983902] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 504206f9332d4aeca098ed2a7c5aebd6 [ 957.084476] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 957.200367] env[64020]: DEBUG nova.compute.manager [req-ed6f4645-fdc6-43a7-b164-476b605739b4 req-bbd6f211-166a-43e9-9bef-d7d9c658794e service nova] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Received event network-vif-deleted-3df624d0-05e9-4386-b3b2-b6915a730dcb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 957.230866] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 95d9560b110a4be08d571294211c578d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 957.264193] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95d9560b110a4be08d571294211c578d [ 957.279025] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-a2cd396e-bd03-4c51-8ec6-cd24d7541103" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 957.279406] env[64020]: DEBUG nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 957.279590] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 957.279862] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8a15b55b-6945-4325-9b3d-fd857e5988a6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 957.289332] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-980998c8-6a8f-4b76-8fee-78f0ca06d472 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 957.310244] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a2cd396e-bd03-4c51-8ec6-cd24d7541103 could not be found. [ 957.310420] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 957.310598] env[64020]: INFO nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Took 0.03 seconds to destroy the instance on the hypervisor. [ 957.310831] env[64020]: DEBUG oslo.service.loopingcall [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 957.311038] env[64020]: DEBUG nova.compute.manager [-] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 957.311129] env[64020]: DEBUG nova.network.neutron [-] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 957.324967] env[64020]: DEBUG nova.network.neutron [-] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 957.325438] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg e6a6f95fd9c94edb836c6afeb4db1bcd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 957.332401] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6a6f95fd9c94edb836c6afeb4db1bcd [ 957.478299] env[64020]: DEBUG nova.scheduler.client.report [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 957.480828] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 66e947e918704503a8a35e265d0294ec in queue reply_57893177120949e6a93cb88e15cd42b4 [ 957.492491] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66e947e918704503a8a35e265d0294ec [ 957.734647] env[64020]: DEBUG nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 957.761111] env[64020]: DEBUG nova.virt.hardware [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 957.761505] env[64020]: DEBUG nova.virt.hardware [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 957.761725] env[64020]: DEBUG nova.virt.hardware [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 957.761967] env[64020]: DEBUG nova.virt.hardware [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 957.762171] env[64020]: DEBUG nova.virt.hardware [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 957.762507] env[64020]: DEBUG nova.virt.hardware [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 957.762798] env[64020]: DEBUG nova.virt.hardware [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 957.763042] env[64020]: DEBUG nova.virt.hardware [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 957.763276] env[64020]: DEBUG nova.virt.hardware [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 957.763497] env[64020]: DEBUG nova.virt.hardware [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 957.763725] env[64020]: DEBUG nova.virt.hardware [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 957.764615] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f714bec-aa66-4bb3-9b61-467d8eed02a0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 957.773205] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f185dac9-beeb-46a6-bd2e-7525f2a29823 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 957.777956] env[64020]: ERROR nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3e907ae1-99ed-4439-863b-3142d5834117, please check neutron logs for more information. [ 957.777956] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 957.777956] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 957.777956] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 957.777956] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 957.777956] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 957.777956] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 957.777956] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 957.777956] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 957.777956] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 957.777956] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 957.777956] env[64020]: ERROR nova.compute.manager raise self.value [ 957.777956] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 957.777956] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 957.777956] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 957.777956] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 957.778392] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 957.778392] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 957.778392] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3e907ae1-99ed-4439-863b-3142d5834117, please check neutron logs for more information. [ 957.778392] env[64020]: ERROR nova.compute.manager [ 957.778392] env[64020]: Traceback (most recent call last): [ 957.778527] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 957.778527] env[64020]: listener.cb(fileno) [ 957.778527] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 957.778527] env[64020]: result = function(*args, **kwargs) [ 957.778527] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 957.778527] env[64020]: return func(*args, **kwargs) [ 957.778527] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 957.778527] env[64020]: raise e [ 957.778527] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 957.778527] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 957.778527] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 957.778527] env[64020]: created_port_ids = self._update_ports_for_instance( [ 957.778527] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 957.778527] env[64020]: with excutils.save_and_reraise_exception(): [ 957.778527] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 957.778527] env[64020]: self.force_reraise() [ 957.778527] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 957.778527] env[64020]: raise self.value [ 957.778527] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 957.778527] env[64020]: updated_port = self._update_port( [ 957.778527] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 957.778527] env[64020]: _ensure_no_port_binding_failure(port) [ 957.778527] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 957.778527] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 957.778527] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 3e907ae1-99ed-4439-863b-3142d5834117, please check neutron logs for more information. [ 957.779215] env[64020]: Removing descriptor: 16 [ 957.788726] env[64020]: ERROR nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3e907ae1-99ed-4439-863b-3142d5834117, please check neutron logs for more information. [ 957.788726] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Traceback (most recent call last): [ 957.788726] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 957.788726] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] yield resources [ 957.788726] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 957.788726] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] self.driver.spawn(context, instance, image_meta, [ 957.788726] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 957.788726] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 957.788726] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 957.788726] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] vm_ref = self.build_virtual_machine(instance, [ 957.788726] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] vif_infos = vmwarevif.get_vif_info(self._session, [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] for vif in network_info: [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] return self._sync_wrapper(fn, *args, **kwargs) [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] self.wait() [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] self[:] = self._gt.wait() [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] return self._exit_event.wait() [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 957.789052] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] current.throw(*self._exc) [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] result = function(*args, **kwargs) [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] return func(*args, **kwargs) [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] raise e [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] nwinfo = self.network_api.allocate_for_instance( [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] created_port_ids = self._update_ports_for_instance( [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] with excutils.save_and_reraise_exception(): [ 957.789448] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 957.789812] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] self.force_reraise() [ 957.789812] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 957.789812] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] raise self.value [ 957.789812] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 957.789812] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] updated_port = self._update_port( [ 957.789812] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 957.789812] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] _ensure_no_port_binding_failure(port) [ 957.789812] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 957.789812] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] raise exception.PortBindingFailed(port_id=port['id']) [ 957.789812] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] nova.exception.PortBindingFailed: Binding failed for port 3e907ae1-99ed-4439-863b-3142d5834117, please check neutron logs for more information. [ 957.789812] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] [ 957.789812] env[64020]: INFO nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Terminating instance [ 957.792134] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquiring lock "refresh_cache-5f485177-eb28-417d-a74c-0e0f30ea7ce2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 957.792373] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquired lock "refresh_cache-5f485177-eb28-417d-a74c-0e0f30ea7ce2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 957.792602] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 957.793060] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg a2fdb169de844a32881eea4d3d525e51 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 957.799889] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2fdb169de844a32881eea4d3d525e51 [ 957.828858] env[64020]: DEBUG nova.network.neutron [-] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 957.828858] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7bf051413cd44ad588d92e876a785537 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 957.836585] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7bf051413cd44ad588d92e876a785537 [ 957.983660] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.271s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 957.986574] env[64020]: DEBUG nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 957.986574] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 99286976d7354886b77136b0d881f6b6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 957.987059] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.119s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 957.988929] env[64020]: INFO nova.compute.claims [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 957.990908] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 05261a47146e45d9bd890b9fc1df5aed in queue reply_57893177120949e6a93cb88e15cd42b4 [ 958.016997] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99286976d7354886b77136b0d881f6b6 [ 958.028033] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05261a47146e45d9bd890b9fc1df5aed [ 958.330790] env[64020]: INFO nova.compute.manager [-] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Took 1.02 seconds to deallocate network for instance. [ 958.333469] env[64020]: DEBUG nova.compute.claims [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 958.333820] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 958.339112] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 958.441884] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 958.442407] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 5f8637e6af6c4baaa4f6166480c5467f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 958.450883] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5f8637e6af6c4baaa4f6166480c5467f [ 958.494281] env[64020]: DEBUG nova.compute.utils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 958.494281] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 69b691caf45f48ed84379dd094ae73db in queue reply_57893177120949e6a93cb88e15cd42b4 [ 958.496321] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg d23ae8f10801423ebed36751535682d9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 958.497206] env[64020]: DEBUG nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 958.497423] env[64020]: DEBUG nova.network.neutron [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 958.503581] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d23ae8f10801423ebed36751535682d9 [ 958.504961] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69b691caf45f48ed84379dd094ae73db [ 958.533200] env[64020]: DEBUG nova.policy [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '29faa85bd7904cd985fb32f53b533c5b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '525215939f6741cea749e7d9e53f379e', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 958.803413] env[64020]: DEBUG nova.network.neutron [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Successfully created port: d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 958.945280] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Releasing lock "refresh_cache-5f485177-eb28-417d-a74c-0e0f30ea7ce2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 958.945711] env[64020]: DEBUG nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 958.946153] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 958.946456] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-dede1236-1815-4c25-b237-5441323372ed {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 958.955824] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90127a5f-7f0c-40ad-ae6e-04139cc48185 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 958.978518] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5f485177-eb28-417d-a74c-0e0f30ea7ce2 could not be found. [ 958.978733] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 958.978898] env[64020]: INFO nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Took 0.03 seconds to destroy the instance on the hypervisor. [ 958.979135] env[64020]: DEBUG oslo.service.loopingcall [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 958.979348] env[64020]: DEBUG nova.compute.manager [-] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 958.979600] env[64020]: DEBUG nova.network.neutron [-] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 958.997812] env[64020]: DEBUG nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 958.999465] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 380c3fd14ce1495197e79cc277df92d8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 959.035384] env[64020]: DEBUG nova.network.neutron [-] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 959.037299] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 380c3fd14ce1495197e79cc277df92d8 [ 959.179450] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fb90b94-8ac9-48a0-9610-159b8728f477 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.187693] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c4357ed-008c-4eac-802e-d9e9d2a75365 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.225693] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 9aa7e4e7a08040b0aa8fc58056a82fe2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 959.226913] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c65faf1-72fb-40dc-90a7-eda2e714ae23 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.231720] env[64020]: DEBUG nova.compute.manager [req-296f7f4c-e7a8-4073-a762-b4c0f10a7416 req-6da6df24-93ab-42a8-88d5-661dc8b0c948 service nova] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Received event network-changed-3e907ae1-99ed-4439-863b-3142d5834117 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 959.231791] env[64020]: DEBUG nova.compute.manager [req-296f7f4c-e7a8-4073-a762-b4c0f10a7416 req-6da6df24-93ab-42a8-88d5-661dc8b0c948 service nova] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Refreshing instance network info cache due to event network-changed-3e907ae1-99ed-4439-863b-3142d5834117. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 959.231957] env[64020]: DEBUG oslo_concurrency.lockutils [req-296f7f4c-e7a8-4073-a762-b4c0f10a7416 req-6da6df24-93ab-42a8-88d5-661dc8b0c948 service nova] Acquiring lock "refresh_cache-5f485177-eb28-417d-a74c-0e0f30ea7ce2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 959.232117] env[64020]: DEBUG oslo_concurrency.lockutils [req-296f7f4c-e7a8-4073-a762-b4c0f10a7416 req-6da6df24-93ab-42a8-88d5-661dc8b0c948 service nova] Acquired lock "refresh_cache-5f485177-eb28-417d-a74c-0e0f30ea7ce2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 959.232299] env[64020]: DEBUG nova.network.neutron [req-296f7f4c-e7a8-4073-a762-b4c0f10a7416 req-6da6df24-93ab-42a8-88d5-661dc8b0c948 service nova] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Refreshing network info cache for port 3e907ae1-99ed-4439-863b-3142d5834117 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 959.232695] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-296f7f4c-e7a8-4073-a762-b4c0f10a7416 req-6da6df24-93ab-42a8-88d5-661dc8b0c948 service nova] Expecting reply to msg 81d3785236784cf0a9ce6aa890ad5f56 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 959.234029] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9aa7e4e7a08040b0aa8fc58056a82fe2 [ 959.234747] env[64020]: DEBUG nova.network.neutron [-] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 959.235162] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7f976692440242bfad7cb788e1b5ba85 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 959.240989] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81d3785236784cf0a9ce6aa890ad5f56 [ 959.242686] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0bb8cd7-6c45-4e01-afb3-fd6257d98a54 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.246838] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f976692440242bfad7cb788e1b5ba85 [ 959.257080] env[64020]: DEBUG nova.compute.provider_tree [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 959.257529] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg efa6302e5c1643679af1f21fd7559e4f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 959.267873] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg efa6302e5c1643679af1f21fd7559e4f [ 959.506238] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg d5beccf6f52a4c6ea850ac4c6f5c84ce in queue reply_57893177120949e6a93cb88e15cd42b4 [ 959.539464] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5beccf6f52a4c6ea850ac4c6f5c84ce [ 959.714336] env[64020]: ERROR nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8, please check neutron logs for more information. [ 959.714336] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 959.714336] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 959.714336] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 959.714336] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 959.714336] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 959.714336] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 959.714336] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 959.714336] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 959.714336] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 959.714336] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 959.714336] env[64020]: ERROR nova.compute.manager raise self.value [ 959.714336] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 959.714336] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 959.714336] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 959.714336] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 959.714792] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 959.714792] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 959.714792] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8, please check neutron logs for more information. [ 959.714792] env[64020]: ERROR nova.compute.manager [ 959.714792] env[64020]: Traceback (most recent call last): [ 959.714792] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 959.714792] env[64020]: listener.cb(fileno) [ 959.714792] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 959.714792] env[64020]: result = function(*args, **kwargs) [ 959.714792] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 959.714792] env[64020]: return func(*args, **kwargs) [ 959.714792] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 959.714792] env[64020]: raise e [ 959.714792] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 959.714792] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 959.714792] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 959.714792] env[64020]: created_port_ids = self._update_ports_for_instance( [ 959.714792] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 959.714792] env[64020]: with excutils.save_and_reraise_exception(): [ 959.714792] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 959.714792] env[64020]: self.force_reraise() [ 959.714792] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 959.714792] env[64020]: raise self.value [ 959.714792] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 959.714792] env[64020]: updated_port = self._update_port( [ 959.714792] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 959.714792] env[64020]: _ensure_no_port_binding_failure(port) [ 959.714792] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 959.714792] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 959.715659] env[64020]: nova.exception.PortBindingFailed: Binding failed for port d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8, please check neutron logs for more information. [ 959.715659] env[64020]: Removing descriptor: 16 [ 959.738163] env[64020]: INFO nova.compute.manager [-] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Took 0.76 seconds to deallocate network for instance. [ 959.740574] env[64020]: DEBUG nova.compute.claims [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 959.740815] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 959.750570] env[64020]: DEBUG nova.network.neutron [req-296f7f4c-e7a8-4073-a762-b4c0f10a7416 req-6da6df24-93ab-42a8-88d5-661dc8b0c948 service nova] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 959.759806] env[64020]: DEBUG nova.scheduler.client.report [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 959.762215] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg ed3becb9434c48c38c65e955b063375b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 959.773152] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed3becb9434c48c38c65e955b063375b [ 959.830691] env[64020]: DEBUG nova.network.neutron [req-296f7f4c-e7a8-4073-a762-b4c0f10a7416 req-6da6df24-93ab-42a8-88d5-661dc8b0c948 service nova] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 959.831382] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-296f7f4c-e7a8-4073-a762-b4c0f10a7416 req-6da6df24-93ab-42a8-88d5-661dc8b0c948 service nova] Expecting reply to msg b9104d17813e454e8a9f911bd71f9b35 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 959.840306] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9104d17813e454e8a9f911bd71f9b35 [ 960.009963] env[64020]: DEBUG nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 960.035463] env[64020]: DEBUG nova.virt.hardware [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 960.035714] env[64020]: DEBUG nova.virt.hardware [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 960.035906] env[64020]: DEBUG nova.virt.hardware [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 960.036072] env[64020]: DEBUG nova.virt.hardware [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 960.036220] env[64020]: DEBUG nova.virt.hardware [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 960.036364] env[64020]: DEBUG nova.virt.hardware [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 960.036570] env[64020]: DEBUG nova.virt.hardware [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 960.036726] env[64020]: DEBUG nova.virt.hardware [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 960.036889] env[64020]: DEBUG nova.virt.hardware [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 960.037045] env[64020]: DEBUG nova.virt.hardware [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 960.037215] env[64020]: DEBUG nova.virt.hardware [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 960.038099] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ddd3824a-9fd9-43f3-9adf-98f231d66549 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 960.046551] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81c55184-3656-4413-8706-047293fe8311 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 960.059763] env[64020]: ERROR nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8, please check neutron logs for more information. [ 960.059763] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Traceback (most recent call last): [ 960.059763] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 960.059763] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] yield resources [ 960.059763] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 960.059763] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] self.driver.spawn(context, instance, image_meta, [ 960.059763] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 960.059763] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 960.059763] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 960.059763] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] vm_ref = self.build_virtual_machine(instance, [ 960.059763] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] vif_infos = vmwarevif.get_vif_info(self._session, [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] for vif in network_info: [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] return self._sync_wrapper(fn, *args, **kwargs) [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] self.wait() [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] self[:] = self._gt.wait() [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] return self._exit_event.wait() [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 960.060150] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] current.throw(*self._exc) [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] result = function(*args, **kwargs) [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] return func(*args, **kwargs) [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] raise e [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] nwinfo = self.network_api.allocate_for_instance( [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] created_port_ids = self._update_ports_for_instance( [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] with excutils.save_and_reraise_exception(): [ 960.060523] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 960.060933] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] self.force_reraise() [ 960.060933] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 960.060933] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] raise self.value [ 960.060933] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 960.060933] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] updated_port = self._update_port( [ 960.060933] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 960.060933] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] _ensure_no_port_binding_failure(port) [ 960.060933] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 960.060933] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] raise exception.PortBindingFailed(port_id=port['id']) [ 960.060933] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] nova.exception.PortBindingFailed: Binding failed for port d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8, please check neutron logs for more information. [ 960.060933] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] [ 960.060933] env[64020]: INFO nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Terminating instance [ 960.062162] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "refresh_cache-c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 960.062365] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquired lock "refresh_cache-c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 960.062596] env[64020]: DEBUG nova.network.neutron [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 960.063016] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 019be8e3884c46a7863a86ca6c8b1075 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 960.071298] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 019be8e3884c46a7863a86ca6c8b1075 [ 960.264504] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.277s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 960.265041] env[64020]: DEBUG nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 960.266815] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 9fc9aae5b34040a38443b40cb185ca3c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 960.267841] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 25.980s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 960.269932] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 1b645d3fd83d4ca8b9e714c4868a0c76 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 960.298848] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9fc9aae5b34040a38443b40cb185ca3c [ 960.299440] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b645d3fd83d4ca8b9e714c4868a0c76 [ 960.333610] env[64020]: DEBUG oslo_concurrency.lockutils [req-296f7f4c-e7a8-4073-a762-b4c0f10a7416 req-6da6df24-93ab-42a8-88d5-661dc8b0c948 service nova] Releasing lock "refresh_cache-5f485177-eb28-417d-a74c-0e0f30ea7ce2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 960.333861] env[64020]: DEBUG nova.compute.manager [req-296f7f4c-e7a8-4073-a762-b4c0f10a7416 req-6da6df24-93ab-42a8-88d5-661dc8b0c948 service nova] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Received event network-vif-deleted-3e907ae1-99ed-4439-863b-3142d5834117 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 960.578903] env[64020]: DEBUG nova.network.neutron [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 960.666616] env[64020]: DEBUG nova.network.neutron [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 960.667143] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 73fa6a0d7a344e838b492a301e46057b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 960.676116] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 73fa6a0d7a344e838b492a301e46057b [ 960.772714] env[64020]: DEBUG nova.compute.utils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 960.773327] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 89efbd4bc6e849e69a319e0e937fd824 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 960.777130] env[64020]: DEBUG nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 960.777594] env[64020]: DEBUG nova.network.neutron [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 960.783238] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89efbd4bc6e849e69a319e0e937fd824 [ 960.934001] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dee3b69e-4793-44c9-9bbe-4172f0207f2c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 960.942367] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f407b61-8d39-4fe0-81d2-ce579467a3bc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 960.974687] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2dfee97b-c202-472c-8798-74daea555e85 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 960.979803] env[64020]: DEBUG nova.policy [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e6417779415348f2b74727cf41c77c54', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'fbebd163ed084de0a8a9f6ccc620ab6a', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 960.984263] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-408ef07e-538c-4440-8a2b-5916b739d4f9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 960.997141] env[64020]: DEBUG nova.compute.provider_tree [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 960.997623] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg bd488a288fe644c69c8479c052a1423a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 961.004554] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bd488a288fe644c69c8479c052a1423a [ 961.169685] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Releasing lock "refresh_cache-c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 961.170372] env[64020]: DEBUG nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 961.170686] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 961.171073] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8107f30f-eb9f-41f5-be26-8aded061eb3d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.180987] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7649246-fd3d-450c-8a1a-4dd80e066c7e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.201968] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c5e780e2-8ef9-461e-bca2-f9d0039ce3c3 could not be found. [ 961.202326] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 961.202562] env[64020]: INFO nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Took 0.03 seconds to destroy the instance on the hypervisor. [ 961.202864] env[64020]: DEBUG oslo.service.loopingcall [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 961.203173] env[64020]: DEBUG nova.compute.manager [-] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 961.203287] env[64020]: DEBUG nova.network.neutron [-] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 961.218188] env[64020]: DEBUG nova.network.neutron [-] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 961.218711] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 42cb6d61a157423695f501fd9bb1bbfd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 961.225547] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42cb6d61a157423695f501fd9bb1bbfd [ 961.263009] env[64020]: DEBUG nova.compute.manager [req-0909c5cf-a8fb-4871-b8ba-2fa415346627 req-05ff5f0f-be9c-4469-bd8d-777bdf611d6e service nova] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Received event network-changed-d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 961.263267] env[64020]: DEBUG nova.compute.manager [req-0909c5cf-a8fb-4871-b8ba-2fa415346627 req-05ff5f0f-be9c-4469-bd8d-777bdf611d6e service nova] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Refreshing instance network info cache due to event network-changed-d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 961.263536] env[64020]: DEBUG oslo_concurrency.lockutils [req-0909c5cf-a8fb-4871-b8ba-2fa415346627 req-05ff5f0f-be9c-4469-bd8d-777bdf611d6e service nova] Acquiring lock "refresh_cache-c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 961.263766] env[64020]: DEBUG oslo_concurrency.lockutils [req-0909c5cf-a8fb-4871-b8ba-2fa415346627 req-05ff5f0f-be9c-4469-bd8d-777bdf611d6e service nova] Acquired lock "refresh_cache-c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 961.264053] env[64020]: DEBUG nova.network.neutron [req-0909c5cf-a8fb-4871-b8ba-2fa415346627 req-05ff5f0f-be9c-4469-bd8d-777bdf611d6e service nova] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Refreshing network info cache for port d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 961.266667] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-0909c5cf-a8fb-4871-b8ba-2fa415346627 req-05ff5f0f-be9c-4469-bd8d-777bdf611d6e service nova] Expecting reply to msg e188e3be5d704b1093606f7f693712f7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 961.276710] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e188e3be5d704b1093606f7f693712f7 [ 961.277460] env[64020]: DEBUG nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 961.279358] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg d5439a798d474052903b00facee7de8f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 961.316856] env[64020]: DEBUG nova.network.neutron [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Successfully created port: 65b815c2-3af4-4a8f-b05d-74efb798ffeb {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 961.323010] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5439a798d474052903b00facee7de8f [ 961.501182] env[64020]: DEBUG nova.scheduler.client.report [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 961.503676] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 43d42c9fa26544218b48d72c5c3328e4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 961.514511] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43d42c9fa26544218b48d72c5c3328e4 [ 961.720625] env[64020]: DEBUG nova.network.neutron [-] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 961.721487] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bc5192193a194cb09d170f0b0397ac25 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 961.730110] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc5192193a194cb09d170f0b0397ac25 [ 961.784075] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 25078735907e457286fb921855b46307 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 961.787074] env[64020]: DEBUG nova.network.neutron [req-0909c5cf-a8fb-4871-b8ba-2fa415346627 req-05ff5f0f-be9c-4469-bd8d-777bdf611d6e service nova] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 961.815063] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 25078735907e457286fb921855b46307 [ 961.863762] env[64020]: DEBUG nova.network.neutron [req-0909c5cf-a8fb-4871-b8ba-2fa415346627 req-05ff5f0f-be9c-4469-bd8d-777bdf611d6e service nova] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 961.864829] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-0909c5cf-a8fb-4871-b8ba-2fa415346627 req-05ff5f0f-be9c-4469-bd8d-777bdf611d6e service nova] Expecting reply to msg 30f19c8d120d4c57b0a550e32bff68a3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 961.872363] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30f19c8d120d4c57b0a550e32bff68a3 [ 962.006836] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.739s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 962.007175] env[64020]: ERROR nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 18a0f809-ed62-4a7d-858d-db5e2199e11d, please check neutron logs for more information. [ 962.007175] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Traceback (most recent call last): [ 962.007175] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 962.007175] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] self.driver.spawn(context, instance, image_meta, [ 962.007175] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 962.007175] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] self._vmops.spawn(context, instance, image_meta, injected_files, [ 962.007175] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 962.007175] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] vm_ref = self.build_virtual_machine(instance, [ 962.007175] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 962.007175] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] vif_infos = vmwarevif.get_vif_info(self._session, [ 962.007175] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] for vif in network_info: [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] return self._sync_wrapper(fn, *args, **kwargs) [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] self.wait() [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] self[:] = self._gt.wait() [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] return self._exit_event.wait() [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] current.throw(*self._exc) [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 962.007473] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] result = function(*args, **kwargs) [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] return func(*args, **kwargs) [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] raise e [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] nwinfo = self.network_api.allocate_for_instance( [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] created_port_ids = self._update_ports_for_instance( [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] with excutils.save_and_reraise_exception(): [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] self.force_reraise() [ 962.007785] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 962.008141] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] raise self.value [ 962.008141] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 962.008141] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] updated_port = self._update_port( [ 962.008141] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 962.008141] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] _ensure_no_port_binding_failure(port) [ 962.008141] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 962.008141] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] raise exception.PortBindingFailed(port_id=port['id']) [ 962.008141] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] nova.exception.PortBindingFailed: Binding failed for port 18a0f809-ed62-4a7d-858d-db5e2199e11d, please check neutron logs for more information. [ 962.008141] env[64020]: ERROR nova.compute.manager [instance: 186cd540-6bb2-407f-9adb-9ed841656713] [ 962.008141] env[64020]: DEBUG nova.compute.utils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Binding failed for port 18a0f809-ed62-4a7d-858d-db5e2199e11d, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 962.009064] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 24.263s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 962.010531] env[64020]: INFO nova.compute.claims [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 962.012263] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg ec926d518c364532b865a83110840ecf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 962.022452] env[64020]: DEBUG nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Build of instance 186cd540-6bb2-407f-9adb-9ed841656713 was re-scheduled: Binding failed for port 18a0f809-ed62-4a7d-858d-db5e2199e11d, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 962.022452] env[64020]: DEBUG nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 962.022452] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquiring lock "refresh_cache-186cd540-6bb2-407f-9adb-9ed841656713" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 962.022452] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquired lock "refresh_cache-186cd540-6bb2-407f-9adb-9ed841656713" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 962.022699] env[64020]: DEBUG nova.network.neutron [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 962.022699] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg a46c67508a3348ca9d85bfcef02a36b5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 962.033054] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a46c67508a3348ca9d85bfcef02a36b5 [ 962.068918] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec926d518c364532b865a83110840ecf [ 962.274041] env[64020]: INFO nova.compute.manager [-] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Took 1.02 seconds to deallocate network for instance. [ 962.274041] env[64020]: DEBUG nova.compute.claims [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 962.274041] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 962.287058] env[64020]: DEBUG nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 962.317854] env[64020]: DEBUG nova.virt.hardware [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 962.318111] env[64020]: DEBUG nova.virt.hardware [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 962.318267] env[64020]: DEBUG nova.virt.hardware [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 962.318448] env[64020]: DEBUG nova.virt.hardware [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 962.318592] env[64020]: DEBUG nova.virt.hardware [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 962.318738] env[64020]: DEBUG nova.virt.hardware [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 962.318948] env[64020]: DEBUG nova.virt.hardware [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 962.319104] env[64020]: DEBUG nova.virt.hardware [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 962.319268] env[64020]: DEBUG nova.virt.hardware [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 962.319427] env[64020]: DEBUG nova.virt.hardware [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 962.319596] env[64020]: DEBUG nova.virt.hardware [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 962.320469] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e78fca21-a207-46ea-ab49-a2781e872ef6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 962.328279] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c256b4e8-6de0-45a3-a5ef-2934db6cfd75 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 962.332627] env[64020]: ERROR nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 65b815c2-3af4-4a8f-b05d-74efb798ffeb, please check neutron logs for more information. [ 962.332627] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 962.332627] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 962.332627] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 962.332627] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 962.332627] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 962.332627] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 962.332627] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 962.332627] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 962.332627] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 962.332627] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 962.332627] env[64020]: ERROR nova.compute.manager raise self.value [ 962.332627] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 962.332627] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 962.332627] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 962.332627] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 962.333107] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 962.333107] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 962.333107] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 65b815c2-3af4-4a8f-b05d-74efb798ffeb, please check neutron logs for more information. [ 962.333107] env[64020]: ERROR nova.compute.manager [ 962.333107] env[64020]: Traceback (most recent call last): [ 962.333107] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 962.333107] env[64020]: listener.cb(fileno) [ 962.333107] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 962.333107] env[64020]: result = function(*args, **kwargs) [ 962.333107] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 962.333107] env[64020]: return func(*args, **kwargs) [ 962.333107] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 962.333107] env[64020]: raise e [ 962.333107] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 962.333107] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 962.333107] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 962.333107] env[64020]: created_port_ids = self._update_ports_for_instance( [ 962.333107] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 962.333107] env[64020]: with excutils.save_and_reraise_exception(): [ 962.333107] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 962.333107] env[64020]: self.force_reraise() [ 962.333107] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 962.333107] env[64020]: raise self.value [ 962.333107] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 962.333107] env[64020]: updated_port = self._update_port( [ 962.333107] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 962.333107] env[64020]: _ensure_no_port_binding_failure(port) [ 962.333107] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 962.333107] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 962.333869] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 65b815c2-3af4-4a8f-b05d-74efb798ffeb, please check neutron logs for more information. [ 962.333869] env[64020]: Removing descriptor: 16 [ 962.344049] env[64020]: ERROR nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 65b815c2-3af4-4a8f-b05d-74efb798ffeb, please check neutron logs for more information. [ 962.344049] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Traceback (most recent call last): [ 962.344049] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 962.344049] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] yield resources [ 962.344049] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 962.344049] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] self.driver.spawn(context, instance, image_meta, [ 962.344049] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 962.344049] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 962.344049] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 962.344049] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] vm_ref = self.build_virtual_machine(instance, [ 962.344049] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] vif_infos = vmwarevif.get_vif_info(self._session, [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] for vif in network_info: [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] return self._sync_wrapper(fn, *args, **kwargs) [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] self.wait() [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] self[:] = self._gt.wait() [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] return self._exit_event.wait() [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 962.344438] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] current.throw(*self._exc) [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] result = function(*args, **kwargs) [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] return func(*args, **kwargs) [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] raise e [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] nwinfo = self.network_api.allocate_for_instance( [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] created_port_ids = self._update_ports_for_instance( [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] with excutils.save_and_reraise_exception(): [ 962.344830] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 962.345190] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] self.force_reraise() [ 962.345190] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 962.345190] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] raise self.value [ 962.345190] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 962.345190] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] updated_port = self._update_port( [ 962.345190] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 962.345190] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] _ensure_no_port_binding_failure(port) [ 962.345190] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 962.345190] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] raise exception.PortBindingFailed(port_id=port['id']) [ 962.345190] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] nova.exception.PortBindingFailed: Binding failed for port 65b815c2-3af4-4a8f-b05d-74efb798ffeb, please check neutron logs for more information. [ 962.345190] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] [ 962.345190] env[64020]: INFO nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Terminating instance [ 962.345654] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquiring lock "refresh_cache-3ad11bab-4531-486f-88c0-3ef7153ea0f5" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 962.345654] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquired lock "refresh_cache-3ad11bab-4531-486f-88c0-3ef7153ea0f5" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 962.345654] env[64020]: DEBUG nova.network.neutron [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 962.346036] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg f794bc43226040228529726654f3834a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 962.358737] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f794bc43226040228529726654f3834a [ 962.365951] env[64020]: DEBUG oslo_concurrency.lockutils [req-0909c5cf-a8fb-4871-b8ba-2fa415346627 req-05ff5f0f-be9c-4469-bd8d-777bdf611d6e service nova] Releasing lock "refresh_cache-c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 962.366169] env[64020]: DEBUG nova.compute.manager [req-0909c5cf-a8fb-4871-b8ba-2fa415346627 req-05ff5f0f-be9c-4469-bd8d-777bdf611d6e service nova] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Received event network-vif-deleted-d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 962.517435] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg f9f649cd0c9d4a189618310f4e2884ea in queue reply_57893177120949e6a93cb88e15cd42b4 [ 962.526899] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f9f649cd0c9d4a189618310f4e2884ea [ 962.540916] env[64020]: DEBUG nova.network.neutron [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 962.650328] env[64020]: DEBUG nova.network.neutron [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 962.650974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 6cb4c31015f64d02a0cab7f53d03446c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 962.660604] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6cb4c31015f64d02a0cab7f53d03446c [ 962.867927] env[64020]: DEBUG nova.network.neutron [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 962.948169] env[64020]: DEBUG nova.network.neutron [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 962.948169] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg b8e8dab530b4406ab610145f62167fc2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 962.948169] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8e8dab530b4406ab610145f62167fc2 [ 963.153722] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Releasing lock "refresh_cache-186cd540-6bb2-407f-9adb-9ed841656713" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 963.154036] env[64020]: DEBUG nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 963.154128] env[64020]: DEBUG nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 963.154293] env[64020]: DEBUG nova.network.neutron [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 963.170625] env[64020]: DEBUG nova.network.neutron [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 963.171168] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 4c264c60770947d8bfe8b8a281a06d17 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 963.177865] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c264c60770947d8bfe8b8a281a06d17 [ 963.219150] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-792207de-5000-437b-ac49-212376251aea {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.233120] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6ed8a51-47bd-4d35-acc4-0f2e65053adf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.266861] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb7bea28-6483-46e9-a73e-5ad0b95de7fa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.274907] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19ef4922-3a86-426e-b7fa-4bd55608460f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.287770] env[64020]: DEBUG nova.compute.provider_tree [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 963.288340] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 00fc9936b64e451885f8a590eec971b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 963.290998] env[64020]: DEBUG nova.compute.manager [req-3a871c61-bcb8-4c08-8f95-1ddabd32d1f3 req-c3f79c76-f1dd-43dc-b6b9-830d33e6bdb9 service nova] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Received event network-changed-65b815c2-3af4-4a8f-b05d-74efb798ffeb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 963.291266] env[64020]: DEBUG nova.compute.manager [req-3a871c61-bcb8-4c08-8f95-1ddabd32d1f3 req-c3f79c76-f1dd-43dc-b6b9-830d33e6bdb9 service nova] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Refreshing instance network info cache due to event network-changed-65b815c2-3af4-4a8f-b05d-74efb798ffeb. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 963.291517] env[64020]: DEBUG oslo_concurrency.lockutils [req-3a871c61-bcb8-4c08-8f95-1ddabd32d1f3 req-c3f79c76-f1dd-43dc-b6b9-830d33e6bdb9 service nova] Acquiring lock "refresh_cache-3ad11bab-4531-486f-88c0-3ef7153ea0f5" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 963.298497] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 00fc9936b64e451885f8a590eec971b3 [ 963.441632] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Releasing lock "refresh_cache-3ad11bab-4531-486f-88c0-3ef7153ea0f5" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 963.448675] env[64020]: DEBUG nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 963.448964] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 963.449354] env[64020]: DEBUG oslo_concurrency.lockutils [req-3a871c61-bcb8-4c08-8f95-1ddabd32d1f3 req-c3f79c76-f1dd-43dc-b6b9-830d33e6bdb9 service nova] Acquired lock "refresh_cache-3ad11bab-4531-486f-88c0-3ef7153ea0f5" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 963.449586] env[64020]: DEBUG nova.network.neutron [req-3a871c61-bcb8-4c08-8f95-1ddabd32d1f3 req-c3f79c76-f1dd-43dc-b6b9-830d33e6bdb9 service nova] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Refreshing network info cache for port 65b815c2-3af4-4a8f-b05d-74efb798ffeb {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 963.450081] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3a871c61-bcb8-4c08-8f95-1ddabd32d1f3 req-c3f79c76-f1dd-43dc-b6b9-830d33e6bdb9 service nova] Expecting reply to msg 0fa3589bba4149be9dbd58b61c89b5b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 963.451290] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3ed88afe-96ae-45cc-bb38-301502080f3a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.458004] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0fa3589bba4149be9dbd58b61c89b5b3 [ 963.460983] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21653ee4-81bd-4389-bf96-71415055c90c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.484198] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3ad11bab-4531-486f-88c0-3ef7153ea0f5 could not be found. [ 963.484449] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 963.484663] env[64020]: INFO nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Took 0.04 seconds to destroy the instance on the hypervisor. [ 963.484928] env[64020]: DEBUG oslo.service.loopingcall [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 963.485195] env[64020]: DEBUG nova.compute.manager [-] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 963.485322] env[64020]: DEBUG nova.network.neutron [-] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 963.499688] env[64020]: DEBUG nova.network.neutron [-] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 963.500158] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5fc1c9a4856f40b6bc9e7b9bbe78ef29 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 963.506588] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5fc1c9a4856f40b6bc9e7b9bbe78ef29 [ 963.673832] env[64020]: DEBUG nova.network.neutron [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 963.674437] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 68979caa6fb4435e88851ba2a2972f32 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 963.682919] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 68979caa6fb4435e88851ba2a2972f32 [ 963.793489] env[64020]: DEBUG nova.scheduler.client.report [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 963.796019] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 8fd809ba0bea465d8b26db31b8a6303a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 963.806861] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8fd809ba0bea465d8b26db31b8a6303a [ 963.968306] env[64020]: DEBUG nova.network.neutron [req-3a871c61-bcb8-4c08-8f95-1ddabd32d1f3 req-c3f79c76-f1dd-43dc-b6b9-830d33e6bdb9 service nova] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 964.002590] env[64020]: DEBUG nova.network.neutron [-] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 964.003056] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7c574c5711a04513ba02513ab9979b93 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 964.013222] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c574c5711a04513ba02513ab9979b93 [ 964.046005] env[64020]: DEBUG nova.network.neutron [req-3a871c61-bcb8-4c08-8f95-1ddabd32d1f3 req-c3f79c76-f1dd-43dc-b6b9-830d33e6bdb9 service nova] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 964.046506] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3a871c61-bcb8-4c08-8f95-1ddabd32d1f3 req-c3f79c76-f1dd-43dc-b6b9-830d33e6bdb9 service nova] Expecting reply to msg f07e2ec000b546b987d39fe16b571ba3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 964.054053] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f07e2ec000b546b987d39fe16b571ba3 [ 964.176704] env[64020]: INFO nova.compute.manager [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: 186cd540-6bb2-407f-9adb-9ed841656713] Took 1.02 seconds to deallocate network for instance. [ 964.178492] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg b0277352059047d78e477fcfb31aa1bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 964.212938] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b0277352059047d78e477fcfb31aa1bc [ 964.298981] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.290s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 964.299586] env[64020]: DEBUG nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 964.301619] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg befd7fb00e024012ba04ee881d762c6f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 964.302723] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 22.508s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 964.304554] env[64020]: INFO nova.compute.claims [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 964.306266] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 9b471f6c95b2480e9a7cf06c5f1e62aa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 964.337930] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg befd7fb00e024012ba04ee881d762c6f [ 964.346850] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b471f6c95b2480e9a7cf06c5f1e62aa [ 964.506286] env[64020]: INFO nova.compute.manager [-] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Took 1.02 seconds to deallocate network for instance. [ 964.508736] env[64020]: DEBUG nova.compute.claims [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 964.508910] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 964.548533] env[64020]: DEBUG oslo_concurrency.lockutils [req-3a871c61-bcb8-4c08-8f95-1ddabd32d1f3 req-c3f79c76-f1dd-43dc-b6b9-830d33e6bdb9 service nova] Releasing lock "refresh_cache-3ad11bab-4531-486f-88c0-3ef7153ea0f5" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 964.548773] env[64020]: DEBUG nova.compute.manager [req-3a871c61-bcb8-4c08-8f95-1ddabd32d1f3 req-c3f79c76-f1dd-43dc-b6b9-830d33e6bdb9 service nova] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Received event network-vif-deleted-65b815c2-3af4-4a8f-b05d-74efb798ffeb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 964.683571] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg e363694a804b4fc5a1cf5ad171f3908f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 964.712702] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e363694a804b4fc5a1cf5ad171f3908f [ 964.809409] env[64020]: DEBUG nova.compute.utils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 964.810136] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 343cef5f68e14dbcb951e90f564dc43a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 964.812210] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 6d33c56b01404f5aa25c360e0825202c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 964.813621] env[64020]: DEBUG nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 964.813786] env[64020]: DEBUG nova.network.neutron [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 964.820627] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 343cef5f68e14dbcb951e90f564dc43a [ 964.823885] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d33c56b01404f5aa25c360e0825202c [ 964.860178] env[64020]: DEBUG nova.policy [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e9a022c35a624ebab962014dc0c72037', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8a8ad74a9f874a21838d224fa7638717', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 965.148935] env[64020]: DEBUG nova.network.neutron [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Successfully created port: bc5e06b8-8c09-40b0-852c-2fb750a365c1 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 965.207409] env[64020]: INFO nova.scheduler.client.report [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Deleted allocations for instance 186cd540-6bb2-407f-9adb-9ed841656713 [ 965.213335] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg aefd7d8746bd4d7e8a9bd247bf8b4c5e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 965.226854] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aefd7d8746bd4d7e8a9bd247bf8b4c5e [ 965.314984] env[64020]: DEBUG nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 965.317882] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg d932f00e5c83416183999bb0a7a4e86e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 965.360990] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d932f00e5c83416183999bb0a7a4e86e [ 965.462238] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a51e629c-ecc4-4268-b2c6-f76e134f7912 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 965.469749] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53cf7d5b-c7f6-4851-bf45-6001c33b99bb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 965.500024] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11c033bc-c6dd-447e-ba26-05bde22cd211 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 965.507175] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cb76381-6eef-4bef-8b49-f17466009e20 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 965.519805] env[64020]: DEBUG nova.compute.provider_tree [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 965.520298] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg fc5bf4953ed74b9083b6bbf25e5ae044 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 965.527161] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc5bf4953ed74b9083b6bbf25e5ae044 [ 965.715337] env[64020]: DEBUG oslo_concurrency.lockutils [None req-5695f9d5-301d-43f4-8b8f-de13335ef85a tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "186cd540-6bb2-407f-9adb-9ed841656713" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 131.140s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 965.715944] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg e9d37193ae854b5b9505e3ea436f506b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 965.745223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9d37193ae854b5b9505e3ea436f506b [ 965.825728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg b33e4074b76f4a7daee49cebebc1bbcb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 965.857643] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b33e4074b76f4a7daee49cebebc1bbcb [ 965.966505] env[64020]: DEBUG nova.compute.manager [req-9bed4304-3076-4039-a4e5-a5d76aac49ff req-4168613b-1d58-4c46-8e80-76d5f0ce5c05 service nova] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Received event network-changed-bc5e06b8-8c09-40b0-852c-2fb750a365c1 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 965.966823] env[64020]: DEBUG nova.compute.manager [req-9bed4304-3076-4039-a4e5-a5d76aac49ff req-4168613b-1d58-4c46-8e80-76d5f0ce5c05 service nova] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Refreshing instance network info cache due to event network-changed-bc5e06b8-8c09-40b0-852c-2fb750a365c1. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 965.967366] env[64020]: DEBUG oslo_concurrency.lockutils [req-9bed4304-3076-4039-a4e5-a5d76aac49ff req-4168613b-1d58-4c46-8e80-76d5f0ce5c05 service nova] Acquiring lock "refresh_cache-50e9cb9c-10fd-466d-9b11-5175d7955ac9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 965.967366] env[64020]: DEBUG oslo_concurrency.lockutils [req-9bed4304-3076-4039-a4e5-a5d76aac49ff req-4168613b-1d58-4c46-8e80-76d5f0ce5c05 service nova] Acquired lock "refresh_cache-50e9cb9c-10fd-466d-9b11-5175d7955ac9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 965.967512] env[64020]: DEBUG nova.network.neutron [req-9bed4304-3076-4039-a4e5-a5d76aac49ff req-4168613b-1d58-4c46-8e80-76d5f0ce5c05 service nova] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Refreshing network info cache for port bc5e06b8-8c09-40b0-852c-2fb750a365c1 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 965.967931] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-9bed4304-3076-4039-a4e5-a5d76aac49ff req-4168613b-1d58-4c46-8e80-76d5f0ce5c05 service nova] Expecting reply to msg b77977d256fb42b590e233cbdc39464b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 965.974464] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b77977d256fb42b590e233cbdc39464b [ 966.023068] env[64020]: DEBUG nova.scheduler.client.report [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 966.025600] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg b93710e63b3641a8842a6b0e6f700e60 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 966.044082] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b93710e63b3641a8842a6b0e6f700e60 [ 966.148728] env[64020]: ERROR nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port bc5e06b8-8c09-40b0-852c-2fb750a365c1, please check neutron logs for more information. [ 966.148728] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 966.148728] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 966.148728] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 966.148728] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 966.148728] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 966.148728] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 966.148728] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 966.148728] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 966.148728] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 966.148728] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 966.148728] env[64020]: ERROR nova.compute.manager raise self.value [ 966.148728] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 966.148728] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 966.148728] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 966.148728] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 966.149262] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 966.149262] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 966.149262] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port bc5e06b8-8c09-40b0-852c-2fb750a365c1, please check neutron logs for more information. [ 966.149262] env[64020]: ERROR nova.compute.manager [ 966.149262] env[64020]: Traceback (most recent call last): [ 966.149262] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 966.149262] env[64020]: listener.cb(fileno) [ 966.149262] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 966.149262] env[64020]: result = function(*args, **kwargs) [ 966.149262] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 966.149262] env[64020]: return func(*args, **kwargs) [ 966.149262] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 966.149262] env[64020]: raise e [ 966.149262] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 966.149262] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 966.149262] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 966.149262] env[64020]: created_port_ids = self._update_ports_for_instance( [ 966.149262] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 966.149262] env[64020]: with excutils.save_and_reraise_exception(): [ 966.149262] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 966.149262] env[64020]: self.force_reraise() [ 966.149262] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 966.149262] env[64020]: raise self.value [ 966.149262] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 966.149262] env[64020]: updated_port = self._update_port( [ 966.149262] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 966.149262] env[64020]: _ensure_no_port_binding_failure(port) [ 966.149262] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 966.149262] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 966.150116] env[64020]: nova.exception.PortBindingFailed: Binding failed for port bc5e06b8-8c09-40b0-852c-2fb750a365c1, please check neutron logs for more information. [ 966.150116] env[64020]: Removing descriptor: 16 [ 966.218218] env[64020]: DEBUG nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 966.220017] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 35dd55ce089c45f1ab460bc98254cdec in queue reply_57893177120949e6a93cb88e15cd42b4 [ 966.250826] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35dd55ce089c45f1ab460bc98254cdec [ 966.328787] env[64020]: DEBUG nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 966.353601] env[64020]: DEBUG nova.virt.hardware [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 966.353844] env[64020]: DEBUG nova.virt.hardware [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 966.353997] env[64020]: DEBUG nova.virt.hardware [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 966.354174] env[64020]: DEBUG nova.virt.hardware [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 966.354388] env[64020]: DEBUG nova.virt.hardware [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 966.354449] env[64020]: DEBUG nova.virt.hardware [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 966.354650] env[64020]: DEBUG nova.virt.hardware [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 966.354804] env[64020]: DEBUG nova.virt.hardware [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 966.354964] env[64020]: DEBUG nova.virt.hardware [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 966.355120] env[64020]: DEBUG nova.virt.hardware [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 966.355287] env[64020]: DEBUG nova.virt.hardware [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 966.356154] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f5131ea-ace4-4f56-a93e-d37437d555ca {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.364210] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a10be9d8-358b-4654-96f6-46bb11496f90 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.378493] env[64020]: ERROR nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port bc5e06b8-8c09-40b0-852c-2fb750a365c1, please check neutron logs for more information. [ 966.378493] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Traceback (most recent call last): [ 966.378493] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 966.378493] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] yield resources [ 966.378493] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 966.378493] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] self.driver.spawn(context, instance, image_meta, [ 966.378493] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 966.378493] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 966.378493] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 966.378493] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] vm_ref = self.build_virtual_machine(instance, [ 966.378493] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] vif_infos = vmwarevif.get_vif_info(self._session, [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] for vif in network_info: [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] return self._sync_wrapper(fn, *args, **kwargs) [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] self.wait() [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] self[:] = self._gt.wait() [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] return self._exit_event.wait() [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 966.378822] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] current.throw(*self._exc) [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] result = function(*args, **kwargs) [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] return func(*args, **kwargs) [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] raise e [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] nwinfo = self.network_api.allocate_for_instance( [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] created_port_ids = self._update_ports_for_instance( [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] with excutils.save_and_reraise_exception(): [ 966.379170] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 966.379522] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] self.force_reraise() [ 966.379522] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 966.379522] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] raise self.value [ 966.379522] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 966.379522] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] updated_port = self._update_port( [ 966.379522] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 966.379522] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] _ensure_no_port_binding_failure(port) [ 966.379522] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 966.379522] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] raise exception.PortBindingFailed(port_id=port['id']) [ 966.379522] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] nova.exception.PortBindingFailed: Binding failed for port bc5e06b8-8c09-40b0-852c-2fb750a365c1, please check neutron logs for more information. [ 966.379522] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] [ 966.379522] env[64020]: INFO nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Terminating instance [ 966.380763] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "refresh_cache-50e9cb9c-10fd-466d-9b11-5175d7955ac9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 966.486027] env[64020]: DEBUG nova.network.neutron [req-9bed4304-3076-4039-a4e5-a5d76aac49ff req-4168613b-1d58-4c46-8e80-76d5f0ce5c05 service nova] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 966.528106] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.225s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 966.528611] env[64020]: DEBUG nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 966.530430] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg b571b371b632465598c399f44c6fe73d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 966.533405] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 21.383s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 966.534740] env[64020]: INFO nova.compute.claims [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 966.536195] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 679897e348734ad393d960c065ba5ef8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 966.561704] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b571b371b632465598c399f44c6fe73d [ 966.568390] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 679897e348734ad393d960c065ba5ef8 [ 966.570134] env[64020]: DEBUG nova.network.neutron [req-9bed4304-3076-4039-a4e5-a5d76aac49ff req-4168613b-1d58-4c46-8e80-76d5f0ce5c05 service nova] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 966.570586] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-9bed4304-3076-4039-a4e5-a5d76aac49ff req-4168613b-1d58-4c46-8e80-76d5f0ce5c05 service nova] Expecting reply to msg 45db615f1e444d55a1d5205db6cac784 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 966.576763] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 45db615f1e444d55a1d5205db6cac784 [ 966.740710] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 967.039340] env[64020]: DEBUG nova.compute.utils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 967.040147] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 491302bed99a45b8a7c6829230e99ebc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 967.042399] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 04afdd70a3b948539d672d69f72ded48 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 967.043274] env[64020]: DEBUG nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 967.043414] env[64020]: DEBUG nova.network.neutron [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 967.051530] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 491302bed99a45b8a7c6829230e99ebc [ 967.052916] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 04afdd70a3b948539d672d69f72ded48 [ 967.072872] env[64020]: DEBUG oslo_concurrency.lockutils [req-9bed4304-3076-4039-a4e5-a5d76aac49ff req-4168613b-1d58-4c46-8e80-76d5f0ce5c05 service nova] Releasing lock "refresh_cache-50e9cb9c-10fd-466d-9b11-5175d7955ac9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 967.073245] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquired lock "refresh_cache-50e9cb9c-10fd-466d-9b11-5175d7955ac9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 967.073414] env[64020]: DEBUG nova.network.neutron [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 967.073818] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg f3cc93f6da834aba8dcb29d983f1a939 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 967.080097] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3cc93f6da834aba8dcb29d983f1a939 [ 967.119175] env[64020]: DEBUG nova.policy [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2e612db51c8a49b3b942ac70e24025f6', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd139e605a6f54506ab7b94cc56578734', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 967.544064] env[64020]: DEBUG nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 967.545919] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg f1b0e265e9484e1194a52e4ba061dbc1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 967.547493] env[64020]: DEBUG nova.network.neutron [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Successfully created port: af0a735b-fb31-43c7-a943-6d3dcdfa83c4 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 967.580955] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f1b0e265e9484e1194a52e4ba061dbc1 [ 967.591512] env[64020]: DEBUG nova.network.neutron [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 967.673159] env[64020]: DEBUG nova.network.neutron [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 967.673685] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 7677fdf249d846409e6f2851eea07732 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 967.683194] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7677fdf249d846409e6f2851eea07732 [ 967.714653] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34476110-66d9-4319-8bf8-2122f90512ba {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.722625] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aded320b-5667-4844-a852-d5b08a548470 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.752628] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a16940d8-87ba-4b18-ae9c-e8d8d1c3c94a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.759463] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8076b7d4-641e-4a1b-8376-9c62b569e025 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.772468] env[64020]: DEBUG nova.compute.provider_tree [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 967.772823] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg c3d1c1e9a6864090b93927aa4df21d6f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 967.782719] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3d1c1e9a6864090b93927aa4df21d6f [ 968.056154] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg d6e6d7d41c0747a48b02ff26d3b6019b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 968.098897] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d6e6d7d41c0747a48b02ff26d3b6019b [ 968.129867] env[64020]: DEBUG nova.compute.manager [req-12cf4251-56a7-4d71-8ae3-57bb48bd6276 req-a74dbd37-5d62-4776-ae39-f6e503e0064b service nova] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Received event network-vif-deleted-bc5e06b8-8c09-40b0-852c-2fb750a365c1 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 968.175634] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Releasing lock "refresh_cache-50e9cb9c-10fd-466d-9b11-5175d7955ac9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 968.176077] env[64020]: DEBUG nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 968.176272] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 968.176570] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-259f6751-5abe-4b8d-8720-8c0107829f9b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 968.185619] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9dec13f4-d4c7-433b-bf20-1d74f5eff4c8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 968.206862] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 50e9cb9c-10fd-466d-9b11-5175d7955ac9 could not be found. [ 968.207075] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 968.207251] env[64020]: INFO nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Took 0.03 seconds to destroy the instance on the hypervisor. [ 968.207489] env[64020]: DEBUG oslo.service.loopingcall [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 968.207707] env[64020]: DEBUG nova.compute.manager [-] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 968.207795] env[64020]: DEBUG nova.network.neutron [-] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 968.231737] env[64020]: DEBUG nova.network.neutron [-] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 968.232283] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 2c3d611c786f4885a8ee1523a7b27742 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 968.240181] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c3d611c786f4885a8ee1523a7b27742 [ 968.275765] env[64020]: DEBUG nova.scheduler.client.report [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 968.278094] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg be2ce482618645a38af74347e9718184 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 968.291034] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be2ce482618645a38af74347e9718184 [ 968.559956] env[64020]: DEBUG nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 968.591618] env[64020]: DEBUG nova.virt.hardware [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 968.591873] env[64020]: DEBUG nova.virt.hardware [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 968.592039] env[64020]: DEBUG nova.virt.hardware [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 968.592249] env[64020]: DEBUG nova.virt.hardware [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 968.592404] env[64020]: DEBUG nova.virt.hardware [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 968.592549] env[64020]: DEBUG nova.virt.hardware [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 968.592751] env[64020]: DEBUG nova.virt.hardware [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 968.592906] env[64020]: DEBUG nova.virt.hardware [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 968.593067] env[64020]: DEBUG nova.virt.hardware [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 968.593223] env[64020]: DEBUG nova.virt.hardware [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 968.593392] env[64020]: DEBUG nova.virt.hardware [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 968.594233] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5071e9e9-0fa8-4e0a-806f-a45f5eac4155 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 968.601928] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59e95a0a-e50a-4110-b1ac-fc59665b656d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 968.734819] env[64020]: DEBUG nova.network.neutron [-] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 968.735318] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg aed3f909384d40a483e286540167e515 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 968.743656] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aed3f909384d40a483e286540167e515 [ 968.780683] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.247s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 968.781544] env[64020]: DEBUG nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 968.783205] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 2e59bc6835574f369f23dbb25dd2d1bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 968.784196] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 16.758s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 968.784365] env[64020]: DEBUG nova.objects.instance [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 15815ad0-f79a-4031-b625-22c5f93dc2c1] Trying to apply a migration context that does not seem to be set for this instance {{(pid=64020) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 968.785750] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg eb596c02c5a84d92b8dc8b69ced0b263 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 968.823881] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb596c02c5a84d92b8dc8b69ced0b263 [ 968.835772] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2e59bc6835574f369f23dbb25dd2d1bc [ 969.154269] env[64020]: ERROR nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port af0a735b-fb31-43c7-a943-6d3dcdfa83c4, please check neutron logs for more information. [ 969.154269] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 969.154269] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 969.154269] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 969.154269] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 969.154269] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 969.154269] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 969.154269] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 969.154269] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 969.154269] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 969.154269] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 969.154269] env[64020]: ERROR nova.compute.manager raise self.value [ 969.154269] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 969.154269] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 969.154269] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 969.154269] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 969.154834] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 969.154834] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 969.154834] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port af0a735b-fb31-43c7-a943-6d3dcdfa83c4, please check neutron logs for more information. [ 969.154834] env[64020]: ERROR nova.compute.manager [ 969.154834] env[64020]: Traceback (most recent call last): [ 969.154834] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 969.154834] env[64020]: listener.cb(fileno) [ 969.154834] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 969.154834] env[64020]: result = function(*args, **kwargs) [ 969.154834] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 969.154834] env[64020]: return func(*args, **kwargs) [ 969.154834] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 969.154834] env[64020]: raise e [ 969.154834] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 969.154834] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 969.154834] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 969.154834] env[64020]: created_port_ids = self._update_ports_for_instance( [ 969.154834] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 969.154834] env[64020]: with excutils.save_and_reraise_exception(): [ 969.154834] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 969.154834] env[64020]: self.force_reraise() [ 969.154834] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 969.154834] env[64020]: raise self.value [ 969.154834] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 969.154834] env[64020]: updated_port = self._update_port( [ 969.154834] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 969.154834] env[64020]: _ensure_no_port_binding_failure(port) [ 969.154834] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 969.154834] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 969.155689] env[64020]: nova.exception.PortBindingFailed: Binding failed for port af0a735b-fb31-43c7-a943-6d3dcdfa83c4, please check neutron logs for more information. [ 969.155689] env[64020]: Removing descriptor: 16 [ 969.155689] env[64020]: ERROR nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port af0a735b-fb31-43c7-a943-6d3dcdfa83c4, please check neutron logs for more information. [ 969.155689] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Traceback (most recent call last): [ 969.155689] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 969.155689] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] yield resources [ 969.155689] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 969.155689] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] self.driver.spawn(context, instance, image_meta, [ 969.155689] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 969.155689] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 969.155689] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 969.155689] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] vm_ref = self.build_virtual_machine(instance, [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] vif_infos = vmwarevif.get_vif_info(self._session, [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] for vif in network_info: [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] return self._sync_wrapper(fn, *args, **kwargs) [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] self.wait() [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] self[:] = self._gt.wait() [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] return self._exit_event.wait() [ 969.156077] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] result = hub.switch() [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] return self.greenlet.switch() [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] result = function(*args, **kwargs) [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] return func(*args, **kwargs) [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] raise e [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] nwinfo = self.network_api.allocate_for_instance( [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 969.156510] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] created_port_ids = self._update_ports_for_instance( [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] with excutils.save_and_reraise_exception(): [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] self.force_reraise() [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] raise self.value [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] updated_port = self._update_port( [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] _ensure_no_port_binding_failure(port) [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 969.156904] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] raise exception.PortBindingFailed(port_id=port['id']) [ 969.157265] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] nova.exception.PortBindingFailed: Binding failed for port af0a735b-fb31-43c7-a943-6d3dcdfa83c4, please check neutron logs for more information. [ 969.157265] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] [ 969.157265] env[64020]: INFO nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Terminating instance [ 969.157447] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "refresh_cache-bf317705-3bf8-4522-b41a-fc023bf766d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 969.157601] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquired lock "refresh_cache-bf317705-3bf8-4522-b41a-fc023bf766d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 969.157761] env[64020]: DEBUG nova.network.neutron [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 969.158167] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 8c246621b92349e5ab8d85ed49145c73 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 969.167466] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8c246621b92349e5ab8d85ed49145c73 [ 969.237837] env[64020]: INFO nova.compute.manager [-] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Took 1.03 seconds to deallocate network for instance. [ 969.240843] env[64020]: DEBUG nova.compute.claims [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 969.241108] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 969.293452] env[64020]: DEBUG nova.compute.utils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 969.294123] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 89623cdb64a94d75a2bbd43e348086f2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 969.297194] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 447c3ad70b33479dba7580a6a6f9300c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 969.298178] env[64020]: DEBUG nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 969.298334] env[64020]: DEBUG nova.network.neutron [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 969.303852] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 447c3ad70b33479dba7580a6a6f9300c [ 969.305351] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89623cdb64a94d75a2bbd43e348086f2 [ 969.368432] env[64020]: DEBUG nova.policy [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7816055398664d03aea78422d7de6c5a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'b6f90b4ed1e44348a056acfd1c4567b6', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 969.664923] env[64020]: DEBUG nova.network.neutron [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Successfully created port: b99f58dd-63de-41a1-853f-7ee402bf6c3c {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 969.674356] env[64020]: DEBUG nova.network.neutron [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 969.750430] env[64020]: DEBUG nova.network.neutron [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 969.750974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg feca88b88904431d882b60e079123c04 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 969.758767] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg feca88b88904431d882b60e079123c04 [ 969.799411] env[64020]: DEBUG nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 969.801177] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 2d08c600a8bf4e179abb2be5e67f92fb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 969.802834] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.019s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 969.803149] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a685d277-370c-4f1f-8ef6-c9127853a8f4 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg ed694cb84d8b4b8a83973d0a16d54640 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 969.806314] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.047s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 969.806526] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg ae96ecf425c4483ba34a61142356e25a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 969.821916] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed694cb84d8b4b8a83973d0a16d54640 [ 969.863342] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d08c600a8bf4e179abb2be5e67f92fb [ 969.866336] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae96ecf425c4483ba34a61142356e25a [ 969.942291] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquiring lock "f616a54a-647c-4454-967c-e0ef50552720" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 969.942291] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "f616a54a-647c-4454-967c-e0ef50552720" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 970.155436] env[64020]: DEBUG nova.compute.manager [req-d6de7af4-0890-4af7-995f-adcdd5b822ef req-3a962ff6-497e-477b-8d62-63801afe65d8 service nova] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Received event network-changed-af0a735b-fb31-43c7-a943-6d3dcdfa83c4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 970.155633] env[64020]: DEBUG nova.compute.manager [req-d6de7af4-0890-4af7-995f-adcdd5b822ef req-3a962ff6-497e-477b-8d62-63801afe65d8 service nova] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Refreshing instance network info cache due to event network-changed-af0a735b-fb31-43c7-a943-6d3dcdfa83c4. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 970.155819] env[64020]: DEBUG oslo_concurrency.lockutils [req-d6de7af4-0890-4af7-995f-adcdd5b822ef req-3a962ff6-497e-477b-8d62-63801afe65d8 service nova] Acquiring lock "refresh_cache-bf317705-3bf8-4522-b41a-fc023bf766d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 970.253098] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Releasing lock "refresh_cache-bf317705-3bf8-4522-b41a-fc023bf766d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 970.253544] env[64020]: DEBUG nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 970.253738] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 970.254049] env[64020]: DEBUG oslo_concurrency.lockutils [req-d6de7af4-0890-4af7-995f-adcdd5b822ef req-3a962ff6-497e-477b-8d62-63801afe65d8 service nova] Acquired lock "refresh_cache-bf317705-3bf8-4522-b41a-fc023bf766d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 970.254216] env[64020]: DEBUG nova.network.neutron [req-d6de7af4-0890-4af7-995f-adcdd5b822ef req-3a962ff6-497e-477b-8d62-63801afe65d8 service nova] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Refreshing network info cache for port af0a735b-fb31-43c7-a943-6d3dcdfa83c4 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 970.254645] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-d6de7af4-0890-4af7-995f-adcdd5b822ef req-3a962ff6-497e-477b-8d62-63801afe65d8 service nova] Expecting reply to msg 2220a1a7557341629c39af97da7f235e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 970.255508] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-48853424-3584-41a1-9b4a-0409eda34e53 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.261464] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2220a1a7557341629c39af97da7f235e [ 970.265664] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f99d1e31-16eb-4f17-9295-b740f52464e6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.287428] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance bf317705-3bf8-4522-b41a-fc023bf766d0 could not be found. [ 970.287643] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 970.287880] env[64020]: INFO nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Took 0.03 seconds to destroy the instance on the hypervisor. [ 970.288181] env[64020]: DEBUG oslo.service.loopingcall [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 970.288462] env[64020]: DEBUG nova.compute.manager [-] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 970.288569] env[64020]: DEBUG nova.network.neutron [-] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 970.302927] env[64020]: DEBUG nova.network.neutron [-] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 970.303395] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 80dfca90f51649dc889e7d9db586bede in queue reply_57893177120949e6a93cb88e15cd42b4 [ 970.306295] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 64828e356ef144c58470ef56fab481ed in queue reply_57893177120949e6a93cb88e15cd42b4 [ 970.311001] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80dfca90f51649dc889e7d9db586bede [ 970.346772] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64828e356ef144c58470ef56fab481ed [ 970.522483] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15b19aa8-186a-422d-a3d5-4360afd41617 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.530753] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab7ce3da-97cc-4867-97d4-0c90d785831d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.564734] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a4caf85-e3c8-4851-8d45-da4f95c70f22 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.571901] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ba10463-99a7-43b4-9852-69ae8302714b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.585959] env[64020]: DEBUG nova.compute.provider_tree [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 970.586812] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg a23b8f408342445e906392f22105d3df in queue reply_57893177120949e6a93cb88e15cd42b4 [ 970.594683] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a23b8f408342445e906392f22105d3df [ 970.664595] env[64020]: ERROR nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b99f58dd-63de-41a1-853f-7ee402bf6c3c, please check neutron logs for more information. [ 970.664595] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 970.664595] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 970.664595] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 970.664595] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 970.664595] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 970.664595] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 970.664595] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 970.664595] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 970.664595] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 970.664595] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 970.664595] env[64020]: ERROR nova.compute.manager raise self.value [ 970.664595] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 970.664595] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 970.664595] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 970.664595] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 970.665243] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 970.665243] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 970.665243] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b99f58dd-63de-41a1-853f-7ee402bf6c3c, please check neutron logs for more information. [ 970.665243] env[64020]: ERROR nova.compute.manager [ 970.665243] env[64020]: Traceback (most recent call last): [ 970.665243] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 970.665243] env[64020]: listener.cb(fileno) [ 970.665243] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 970.665243] env[64020]: result = function(*args, **kwargs) [ 970.665243] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 970.665243] env[64020]: return func(*args, **kwargs) [ 970.665243] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 970.665243] env[64020]: raise e [ 970.665243] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 970.665243] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 970.665243] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 970.665243] env[64020]: created_port_ids = self._update_ports_for_instance( [ 970.665243] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 970.665243] env[64020]: with excutils.save_and_reraise_exception(): [ 970.665243] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 970.665243] env[64020]: self.force_reraise() [ 970.665243] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 970.665243] env[64020]: raise self.value [ 970.665243] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 970.665243] env[64020]: updated_port = self._update_port( [ 970.665243] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 970.665243] env[64020]: _ensure_no_port_binding_failure(port) [ 970.665243] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 970.665243] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 970.665982] env[64020]: nova.exception.PortBindingFailed: Binding failed for port b99f58dd-63de-41a1-853f-7ee402bf6c3c, please check neutron logs for more information. [ 970.665982] env[64020]: Removing descriptor: 16 [ 970.772415] env[64020]: DEBUG nova.network.neutron [req-d6de7af4-0890-4af7-995f-adcdd5b822ef req-3a962ff6-497e-477b-8d62-63801afe65d8 service nova] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 970.807597] env[64020]: DEBUG nova.network.neutron [-] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 970.808036] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 88ad0de12c38419c9437a0587c2c3a3d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 970.809615] env[64020]: DEBUG nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 970.816423] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88ad0de12c38419c9437a0587c2c3a3d [ 970.843713] env[64020]: DEBUG nova.virt.hardware [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 970.843949] env[64020]: DEBUG nova.virt.hardware [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 970.844122] env[64020]: DEBUG nova.virt.hardware [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 970.844343] env[64020]: DEBUG nova.virt.hardware [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 970.844449] env[64020]: DEBUG nova.virt.hardware [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 970.844593] env[64020]: DEBUG nova.virt.hardware [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 970.844794] env[64020]: DEBUG nova.virt.hardware [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 970.844948] env[64020]: DEBUG nova.virt.hardware [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 970.845107] env[64020]: DEBUG nova.virt.hardware [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 970.845265] env[64020]: DEBUG nova.virt.hardware [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 970.845432] env[64020]: DEBUG nova.virt.hardware [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 970.846285] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c218f3e9-5950-458d-b78d-87159f47b7c9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.849211] env[64020]: DEBUG nova.network.neutron [req-d6de7af4-0890-4af7-995f-adcdd5b822ef req-3a962ff6-497e-477b-8d62-63801afe65d8 service nova] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 970.849671] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-d6de7af4-0890-4af7-995f-adcdd5b822ef req-3a962ff6-497e-477b-8d62-63801afe65d8 service nova] Expecting reply to msg f98ed4d8d6fc44b2b31208580a2671a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 970.856107] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02388602-c7c3-489b-93ba-2079eba68514 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.859845] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f98ed4d8d6fc44b2b31208580a2671a2 [ 970.869840] env[64020]: ERROR nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b99f58dd-63de-41a1-853f-7ee402bf6c3c, please check neutron logs for more information. [ 970.869840] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Traceback (most recent call last): [ 970.869840] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 970.869840] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] yield resources [ 970.869840] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 970.869840] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] self.driver.spawn(context, instance, image_meta, [ 970.869840] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 970.869840] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 970.869840] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 970.869840] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] vm_ref = self.build_virtual_machine(instance, [ 970.869840] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] vif_infos = vmwarevif.get_vif_info(self._session, [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] for vif in network_info: [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] return self._sync_wrapper(fn, *args, **kwargs) [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] self.wait() [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] self[:] = self._gt.wait() [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] return self._exit_event.wait() [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 970.870272] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] current.throw(*self._exc) [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] result = function(*args, **kwargs) [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] return func(*args, **kwargs) [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] raise e [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] nwinfo = self.network_api.allocate_for_instance( [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] created_port_ids = self._update_ports_for_instance( [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] with excutils.save_and_reraise_exception(): [ 970.870711] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 970.871107] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] self.force_reraise() [ 970.871107] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 970.871107] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] raise self.value [ 970.871107] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 970.871107] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] updated_port = self._update_port( [ 970.871107] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 970.871107] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] _ensure_no_port_binding_failure(port) [ 970.871107] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 970.871107] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] raise exception.PortBindingFailed(port_id=port['id']) [ 970.871107] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] nova.exception.PortBindingFailed: Binding failed for port b99f58dd-63de-41a1-853f-7ee402bf6c3c, please check neutron logs for more information. [ 970.871107] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] [ 970.871107] env[64020]: INFO nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Terminating instance [ 970.872217] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Acquiring lock "refresh_cache-9d199d06-86cb-4d2f-894d-1ce0ef29235f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 970.872390] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Acquired lock "refresh_cache-9d199d06-86cb-4d2f-894d-1ce0ef29235f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 970.872551] env[64020]: DEBUG nova.network.neutron [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 970.872923] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg d213bbb5d8d84be983611a82aae92cb2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 970.878510] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d213bbb5d8d84be983611a82aae92cb2 [ 971.090225] env[64020]: DEBUG nova.scheduler.client.report [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 971.092707] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 47c8fbaefc65451d829207908d0c592a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 971.111080] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47c8fbaefc65451d829207908d0c592a [ 971.313560] env[64020]: INFO nova.compute.manager [-] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Took 1.02 seconds to deallocate network for instance. [ 971.315790] env[64020]: DEBUG nova.compute.claims [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 971.315963] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 971.351614] env[64020]: DEBUG oslo_concurrency.lockutils [req-d6de7af4-0890-4af7-995f-adcdd5b822ef req-3a962ff6-497e-477b-8d62-63801afe65d8 service nova] Releasing lock "refresh_cache-bf317705-3bf8-4522-b41a-fc023bf766d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 971.351834] env[64020]: DEBUG nova.compute.manager [req-d6de7af4-0890-4af7-995f-adcdd5b822ef req-3a962ff6-497e-477b-8d62-63801afe65d8 service nova] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Received event network-vif-deleted-af0a735b-fb31-43c7-a943-6d3dcdfa83c4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 971.389496] env[64020]: DEBUG nova.network.neutron [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 971.473889] env[64020]: DEBUG nova.network.neutron [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 971.474412] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 0ad36f479b674bd697b56db45eddaf21 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 971.482067] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0ad36f479b674bd697b56db45eddaf21 [ 971.596037] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.792s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 971.596538] env[64020]: ERROR nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e3abbc1c-b3d0-47b8-ac47-aff06027311b, please check neutron logs for more information. [ 971.596538] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Traceback (most recent call last): [ 971.596538] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 971.596538] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] self.driver.spawn(context, instance, image_meta, [ 971.596538] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 971.596538] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 971.596538] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 971.596538] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] vm_ref = self.build_virtual_machine(instance, [ 971.596538] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 971.596538] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] vif_infos = vmwarevif.get_vif_info(self._session, [ 971.596538] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] for vif in network_info: [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] return self._sync_wrapper(fn, *args, **kwargs) [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] self.wait() [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] self[:] = self._gt.wait() [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] return self._exit_event.wait() [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] current.throw(*self._exc) [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 971.596863] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] result = function(*args, **kwargs) [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] return func(*args, **kwargs) [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] raise e [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] nwinfo = self.network_api.allocate_for_instance( [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] created_port_ids = self._update_ports_for_instance( [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] with excutils.save_and_reraise_exception(): [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] self.force_reraise() [ 971.597178] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 971.597496] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] raise self.value [ 971.597496] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 971.597496] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] updated_port = self._update_port( [ 971.597496] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 971.597496] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] _ensure_no_port_binding_failure(port) [ 971.597496] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 971.597496] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] raise exception.PortBindingFailed(port_id=port['id']) [ 971.597496] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] nova.exception.PortBindingFailed: Binding failed for port e3abbc1c-b3d0-47b8-ac47-aff06027311b, please check neutron logs for more information. [ 971.597496] env[64020]: ERROR nova.compute.manager [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] [ 971.597496] env[64020]: DEBUG nova.compute.utils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Binding failed for port e3abbc1c-b3d0-47b8-ac47-aff06027311b, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 971.598352] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.194s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 971.600330] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 273ed1c3e4494608866bbf1a48f5c638 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 971.601668] env[64020]: DEBUG nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Build of instance 51b7673b-15ff-4940-9336-b17bc712f0e3 was re-scheduled: Binding failed for port e3abbc1c-b3d0-47b8-ac47-aff06027311b, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 971.602150] env[64020]: DEBUG nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 971.602429] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquiring lock "refresh_cache-51b7673b-15ff-4940-9336-b17bc712f0e3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 971.602593] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Acquired lock "refresh_cache-51b7673b-15ff-4940-9336-b17bc712f0e3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 971.602749] env[64020]: DEBUG nova.network.neutron [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 971.603101] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 44301d47cdd34c4dbc02551f56da79ed in queue reply_57893177120949e6a93cb88e15cd42b4 [ 971.608565] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44301d47cdd34c4dbc02551f56da79ed [ 971.643162] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 273ed1c3e4494608866bbf1a48f5c638 [ 971.976624] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Releasing lock "refresh_cache-9d199d06-86cb-4d2f-894d-1ce0ef29235f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 971.977093] env[64020]: DEBUG nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 971.977197] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 971.977498] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-36e57ec2-4596-4e6c-ba93-c3235652df7d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 971.986781] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7632d1af-268d-4285-8c4b-1aa09dd89ccb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.008228] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9d199d06-86cb-4d2f-894d-1ce0ef29235f could not be found. [ 972.008470] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 972.008656] env[64020]: INFO nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Took 0.03 seconds to destroy the instance on the hypervisor. [ 972.008906] env[64020]: DEBUG oslo.service.loopingcall [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 972.009141] env[64020]: DEBUG nova.compute.manager [-] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 972.009230] env[64020]: DEBUG nova.network.neutron [-] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 972.025724] env[64020]: DEBUG nova.network.neutron [-] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 972.026312] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1f330704d81441b5a0acb64b6d608520 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 972.036472] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f330704d81441b5a0acb64b6d608520 [ 972.130114] env[64020]: DEBUG nova.network.neutron [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 972.178478] env[64020]: DEBUG nova.compute.manager [req-5db69291-4131-414c-9320-a34d6d7605d5 req-f09b74c2-2bfd-4758-b7d6-f7b3c286388a service nova] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Received event network-changed-b99f58dd-63de-41a1-853f-7ee402bf6c3c {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 972.178665] env[64020]: DEBUG nova.compute.manager [req-5db69291-4131-414c-9320-a34d6d7605d5 req-f09b74c2-2bfd-4758-b7d6-f7b3c286388a service nova] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Refreshing instance network info cache due to event network-changed-b99f58dd-63de-41a1-853f-7ee402bf6c3c. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 972.178868] env[64020]: DEBUG oslo_concurrency.lockutils [req-5db69291-4131-414c-9320-a34d6d7605d5 req-f09b74c2-2bfd-4758-b7d6-f7b3c286388a service nova] Acquiring lock "refresh_cache-9d199d06-86cb-4d2f-894d-1ce0ef29235f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 972.179001] env[64020]: DEBUG oslo_concurrency.lockutils [req-5db69291-4131-414c-9320-a34d6d7605d5 req-f09b74c2-2bfd-4758-b7d6-f7b3c286388a service nova] Acquired lock "refresh_cache-9d199d06-86cb-4d2f-894d-1ce0ef29235f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 972.179186] env[64020]: DEBUG nova.network.neutron [req-5db69291-4131-414c-9320-a34d6d7605d5 req-f09b74c2-2bfd-4758-b7d6-f7b3c286388a service nova] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Refreshing network info cache for port b99f58dd-63de-41a1-853f-7ee402bf6c3c {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 972.179678] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-5db69291-4131-414c-9320-a34d6d7605d5 req-f09b74c2-2bfd-4758-b7d6-f7b3c286388a service nova] Expecting reply to msg 1be370f7b0b741d2930a49f09bc1bd1e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 972.186019] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1be370f7b0b741d2930a49f09bc1bd1e [ 972.224314] env[64020]: DEBUG nova.network.neutron [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 972.224787] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg ad4192d8a12e4135b31bdff10cb78829 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 972.232558] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad4192d8a12e4135b31bdff10cb78829 [ 972.273718] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4a0cf55-2282-4b1a-b5ed-9fa2ab257871 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.280924] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c99896ab-ac85-48a9-b9f1-721babb5b9ac {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.311467] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa333b4c-a144-47f1-8ace-2fee262c24e3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.318752] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e56e5e7-4a2c-4be0-b052-0d83b237aa24 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.331301] env[64020]: DEBUG nova.compute.provider_tree [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 972.331810] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 21426d51e84548ee81da4b627b51e0e2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 972.338709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21426d51e84548ee81da4b627b51e0e2 [ 972.528948] env[64020]: DEBUG nova.network.neutron [-] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 972.529456] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 12c7a63caab44cce95d82218af837750 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 972.537705] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 12c7a63caab44cce95d82218af837750 [ 972.698322] env[64020]: DEBUG nova.network.neutron [req-5db69291-4131-414c-9320-a34d6d7605d5 req-f09b74c2-2bfd-4758-b7d6-f7b3c286388a service nova] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 972.728798] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Releasing lock "refresh_cache-51b7673b-15ff-4940-9336-b17bc712f0e3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 972.729088] env[64020]: DEBUG nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 972.729401] env[64020]: DEBUG nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 972.729607] env[64020]: DEBUG nova.network.neutron [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 972.745729] env[64020]: DEBUG nova.network.neutron [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 972.746396] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 934e3a6b86774a4d9c0f54c62464a7dd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 972.753258] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 934e3a6b86774a4d9c0f54c62464a7dd [ 972.775799] env[64020]: DEBUG nova.network.neutron [req-5db69291-4131-414c-9320-a34d6d7605d5 req-f09b74c2-2bfd-4758-b7d6-f7b3c286388a service nova] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 972.776318] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-5db69291-4131-414c-9320-a34d6d7605d5 req-f09b74c2-2bfd-4758-b7d6-f7b3c286388a service nova] Expecting reply to msg 4f95e2cbeb464284923865e43e1c7171 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 972.783993] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f95e2cbeb464284923865e43e1c7171 [ 972.834635] env[64020]: DEBUG nova.scheduler.client.report [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 972.836961] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg ab0e9d5c67d544f9ab0e19a11d73ef4d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 972.847648] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab0e9d5c67d544f9ab0e19a11d73ef4d [ 973.031832] env[64020]: INFO nova.compute.manager [-] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Took 1.02 seconds to deallocate network for instance. [ 973.034523] env[64020]: DEBUG nova.compute.claims [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 973.034708] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 973.248811] env[64020]: DEBUG nova.network.neutron [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 973.249606] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg eae13a074c374d488f9ffc2d8a23f966 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 973.259086] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eae13a074c374d488f9ffc2d8a23f966 [ 973.278902] env[64020]: DEBUG oslo_concurrency.lockutils [req-5db69291-4131-414c-9320-a34d6d7605d5 req-f09b74c2-2bfd-4758-b7d6-f7b3c286388a service nova] Releasing lock "refresh_cache-9d199d06-86cb-4d2f-894d-1ce0ef29235f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 973.279248] env[64020]: DEBUG nova.compute.manager [req-5db69291-4131-414c-9320-a34d6d7605d5 req-f09b74c2-2bfd-4758-b7d6-f7b3c286388a service nova] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Received event network-vif-deleted-b99f58dd-63de-41a1-853f-7ee402bf6c3c {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 973.339402] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.741s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 973.340059] env[64020]: ERROR nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d8633a09-a42e-4059-b375-94f01ce09ebc, please check neutron logs for more information. [ 973.340059] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Traceback (most recent call last): [ 973.340059] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 973.340059] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] self.driver.spawn(context, instance, image_meta, [ 973.340059] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 973.340059] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 973.340059] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 973.340059] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] vm_ref = self.build_virtual_machine(instance, [ 973.340059] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 973.340059] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] vif_infos = vmwarevif.get_vif_info(self._session, [ 973.340059] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] for vif in network_info: [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] return self._sync_wrapper(fn, *args, **kwargs) [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] self.wait() [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] self[:] = self._gt.wait() [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] return self._exit_event.wait() [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] current.throw(*self._exc) [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 973.340413] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] result = function(*args, **kwargs) [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] return func(*args, **kwargs) [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] raise e [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] nwinfo = self.network_api.allocate_for_instance( [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] created_port_ids = self._update_ports_for_instance( [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] with excutils.save_and_reraise_exception(): [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] self.force_reraise() [ 973.340792] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 973.341178] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] raise self.value [ 973.341178] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 973.341178] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] updated_port = self._update_port( [ 973.341178] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 973.341178] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] _ensure_no_port_binding_failure(port) [ 973.341178] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 973.341178] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] raise exception.PortBindingFailed(port_id=port['id']) [ 973.341178] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] nova.exception.PortBindingFailed: Binding failed for port d8633a09-a42e-4059-b375-94f01ce09ebc, please check neutron logs for more information. [ 973.341178] env[64020]: ERROR nova.compute.manager [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] [ 973.341178] env[64020]: DEBUG nova.compute.utils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Binding failed for port d8633a09-a42e-4059-b375-94f01ce09ebc, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 973.342114] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 16.258s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 973.342409] env[64020]: DEBUG nova.objects.instance [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lazy-loading 'resources' on Instance uuid 15815ad0-f79a-4031-b625-22c5f93dc2c1 {{(pid=64020) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 973.342740] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 4ba27fd0301740efa2f31d5daffe070f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 973.344049] env[64020]: DEBUG nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Build of instance f2c7916a-aa0d-4b41-9c02-e2cee84acf4e was re-scheduled: Binding failed for port d8633a09-a42e-4059-b375-94f01ce09ebc, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 973.344481] env[64020]: DEBUG nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 973.344709] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 973.344854] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 973.345014] env[64020]: DEBUG nova.network.neutron [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 973.345406] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 013c5d47c72d4f8fa92a2b0a3e78be03 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 973.349449] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4ba27fd0301740efa2f31d5daffe070f [ 973.354454] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 013c5d47c72d4f8fa92a2b0a3e78be03 [ 973.752876] env[64020]: INFO nova.compute.manager [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] [instance: 51b7673b-15ff-4940-9336-b17bc712f0e3] Took 1.02 seconds to deallocate network for instance. [ 973.754776] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg cfe2e732a0454986ad89b56906a05daf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 973.786761] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfe2e732a0454986ad89b56906a05daf [ 973.863674] env[64020]: DEBUG nova.network.neutron [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 973.947119] env[64020]: DEBUG nova.network.neutron [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 973.947622] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 74837381c65d4988971ac5230c21fffa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 973.955547] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74837381c65d4988971ac5230c21fffa [ 973.988904] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-649f2564-5e60-4d54-89d8-960ef9f0118a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 973.996692] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b3aedff-8aac-45db-b480-d9451c5568d2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.026535] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a699bc4-51d0-4ac4-8baa-dba1d9ec2eaf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.033531] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e9c0b697-ef6c-46ac-9833-23b57d7a44a4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.045785] env[64020]: DEBUG nova.compute.provider_tree [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 974.046259] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 52150e6349624315929a86af9a9fd933 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 974.053442] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52150e6349624315929a86af9a9fd933 [ 974.259341] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg f869e2853342466b87916c785c8b443d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 974.293942] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f869e2853342466b87916c785c8b443d [ 974.452051] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 974.452051] env[64020]: DEBUG nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 974.452051] env[64020]: DEBUG nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 974.452051] env[64020]: DEBUG nova.network.neutron [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 974.465440] env[64020]: DEBUG nova.network.neutron [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 974.465975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg ebddbf96b836485785090f8d701b43e9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 974.472524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ebddbf96b836485785090f8d701b43e9 [ 974.548774] env[64020]: DEBUG nova.scheduler.client.report [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 974.551114] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 920e8a4258de4d42a2ff195e63e21d3b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 974.561394] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 920e8a4258de4d42a2ff195e63e21d3b [ 974.780691] env[64020]: INFO nova.scheduler.client.report [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Deleted allocations for instance 51b7673b-15ff-4940-9336-b17bc712f0e3 [ 974.794823] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Expecting reply to msg 8871eecb578b4d079dfef9306d749d3f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 974.800436] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8871eecb578b4d079dfef9306d749d3f [ 974.967801] env[64020]: DEBUG nova.network.neutron [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 974.968353] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 1e25686e1d7f402882e48a4d9dcee5f7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 974.978695] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e25686e1d7f402882e48a4d9dcee5f7 [ 975.053804] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.712s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.056046] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.722s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 975.057972] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 82a4f87415c94f23a59710cad3404fcb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 975.076479] env[64020]: INFO nova.scheduler.client.report [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Deleted allocations for instance 15815ad0-f79a-4031-b625-22c5f93dc2c1 [ 975.079010] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg f575a96c36fe49ed867b3d1192dec37e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 975.092850] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82a4f87415c94f23a59710cad3404fcb [ 975.113449] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f575a96c36fe49ed867b3d1192dec37e [ 975.288701] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b9086e7d-e7bd-4acc-a51a-4f029db074dd tempest-AttachVolumeShelveTestJSON-343150829 tempest-AttachVolumeShelveTestJSON-343150829-project-member] Lock "51b7673b-15ff-4940-9336-b17bc712f0e3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 139.398s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.289362] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 484543ef097142e8adf5c884956581ad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 975.299238] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 484543ef097142e8adf5c884956581ad [ 975.470869] env[64020]: INFO nova.compute.manager [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: f2c7916a-aa0d-4b41-9c02-e2cee84acf4e] Took 1.02 seconds to deallocate network for instance. [ 975.472823] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg ff41a1a0dba241428cff050efb9b5351 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 975.523072] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff41a1a0dba241428cff050efb9b5351 [ 975.585929] env[64020]: DEBUG oslo_concurrency.lockutils [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "15815ad0-f79a-4031-b625-22c5f93dc2c1" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 22.186s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.586267] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-41d7eff4-eec6-4538-84a2-9f37002baab1 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 919620ca5b8348e58c054942d316e2ef in queue reply_57893177120949e6a93cb88e15cd42b4 [ 975.599087] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 919620ca5b8348e58c054942d316e2ef [ 975.704258] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 3e5fd4b5c5814e858bb89645b32050ea in queue reply_57893177120949e6a93cb88e15cd42b4 [ 975.713754] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e5fd4b5c5814e858bb89645b32050ea [ 975.720561] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09c1089c-b05b-4a69-9a06-61d0a34516c6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 975.728252] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de9d32cb-5361-4085-96b0-eb34ef08c052 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 975.758321] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54a193ed-7611-4737-a26d-959e65dcfb70 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 975.765357] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e15bcb1-487d-46ef-9c16-03c4675f66ce {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 975.777916] env[64020]: DEBUG nova.compute.provider_tree [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 975.778413] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 146ddbbf5bdb40219c26b02ba1e1d3c7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 975.785957] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 146ddbbf5bdb40219c26b02ba1e1d3c7 [ 975.791382] env[64020]: DEBUG nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 975.793134] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg ee78a7cc519b47798fa5d37bd33caa3a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 975.827620] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ee78a7cc519b47798fa5d37bd33caa3a [ 975.977700] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 8a4f60eb46364b3ea89c31d8f35103bd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 976.008643] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a4f60eb46364b3ea89c31d8f35103bd [ 976.205899] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "77801ae9-89db-4dc7-af03-0646af73b121" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 976.206188] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "77801ae9-89db-4dc7-af03-0646af73b121" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 976.206361] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "77801ae9-89db-4dc7-af03-0646af73b121-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 976.206540] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "77801ae9-89db-4dc7-af03-0646af73b121-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 976.206710] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "77801ae9-89db-4dc7-af03-0646af73b121-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 976.209047] env[64020]: INFO nova.compute.manager [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Terminating instance [ 976.210687] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "refresh_cache-77801ae9-89db-4dc7-af03-0646af73b121" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 976.210844] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquired lock "refresh_cache-77801ae9-89db-4dc7-af03-0646af73b121" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 976.211007] env[64020]: DEBUG nova.network.neutron [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 976.211450] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg e5b2899586034068a254e9ad2480a8db in queue reply_57893177120949e6a93cb88e15cd42b4 [ 976.218679] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e5b2899586034068a254e9ad2480a8db [ 976.281688] env[64020]: DEBUG nova.scheduler.client.report [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 976.284181] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 4ae0d64f168c4454b2a1145cea44cddc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 976.299468] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4ae0d64f168c4454b2a1145cea44cddc [ 976.327762] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 976.503102] env[64020]: INFO nova.scheduler.client.report [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Deleted allocations for instance f2c7916a-aa0d-4b41-9c02-e2cee84acf4e [ 976.509898] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg e3e1b737ca97408eb79d840bf8c0cdfa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 976.522371] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3e1b737ca97408eb79d840bf8c0cdfa [ 976.731787] env[64020]: DEBUG nova.network.neutron [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 976.790892] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.731s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 976.790892] env[64020]: ERROR nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3df624d0-05e9-4386-b3b2-b6915a730dcb, please check neutron logs for more information. [ 976.790892] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Traceback (most recent call last): [ 976.790892] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 976.790892] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] self.driver.spawn(context, instance, image_meta, [ 976.790892] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 976.790892] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] self._vmops.spawn(context, instance, image_meta, injected_files, [ 976.790892] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 976.790892] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] vm_ref = self.build_virtual_machine(instance, [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] vif_infos = vmwarevif.get_vif_info(self._session, [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] for vif in network_info: [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] return self._sync_wrapper(fn, *args, **kwargs) [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] self.wait() [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] self[:] = self._gt.wait() [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] return self._exit_event.wait() [ 976.791413] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] current.throw(*self._exc) [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] result = function(*args, **kwargs) [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] return func(*args, **kwargs) [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] raise e [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] nwinfo = self.network_api.allocate_for_instance( [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] created_port_ids = self._update_ports_for_instance( [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 976.791783] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] with excutils.save_and_reraise_exception(): [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] self.force_reraise() [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] raise self.value [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] updated_port = self._update_port( [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] _ensure_no_port_binding_failure(port) [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] raise exception.PortBindingFailed(port_id=port['id']) [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] nova.exception.PortBindingFailed: Binding failed for port 3df624d0-05e9-4386-b3b2-b6915a730dcb, please check neutron logs for more information. [ 976.792148] env[64020]: ERROR nova.compute.manager [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] [ 976.792656] env[64020]: DEBUG nova.compute.utils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Binding failed for port 3df624d0-05e9-4386-b3b2-b6915a730dcb, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 976.792656] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.049s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 976.792656] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 062e25b8fb0749dcbf7d812f111b833e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 976.793039] env[64020]: DEBUG nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Build of instance a2cd396e-bd03-4c51-8ec6-cd24d7541103 was re-scheduled: Binding failed for port 3df624d0-05e9-4386-b3b2-b6915a730dcb, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 976.793354] env[64020]: DEBUG nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 976.793609] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-a2cd396e-bd03-4c51-8ec6-cd24d7541103" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 976.793897] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-a2cd396e-bd03-4c51-8ec6-cd24d7541103" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 976.794052] env[64020]: DEBUG nova.network.neutron [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 976.794449] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg c4692d60e15b4b3fa0bd395d750c465a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 976.815522] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c4692d60e15b4b3fa0bd395d750c465a [ 976.850778] env[64020]: DEBUG nova.network.neutron [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 976.851320] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 3fc5e1bde35843d1b2fb706aa924f0e1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 976.867960] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3fc5e1bde35843d1b2fb706aa924f0e1 [ 976.876033] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 062e25b8fb0749dcbf7d812f111b833e [ 977.011635] env[64020]: DEBUG oslo_concurrency.lockutils [None req-9b60e7b9-bccc-43d5-a277-ad61ce271f86 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "f2c7916a-aa0d-4b41-9c02-e2cee84acf4e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 110.688s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 977.319622] env[64020]: DEBUG nova.network.neutron [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 977.354520] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Releasing lock "refresh_cache-77801ae9-89db-4dc7-af03-0646af73b121" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 977.354955] env[64020]: DEBUG nova.compute.manager [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 977.355144] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 977.356019] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e3f890d-8868-433d-b902-1166aff258bd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 977.363735] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Powering off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 977.366131] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-728e3f43-6e32-4455-834f-cb3a94be5354 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 977.372581] env[64020]: DEBUG oslo_vmware.api [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 977.372581] env[64020]: value = "task-407962" [ 977.372581] env[64020]: _type = "Task" [ 977.372581] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 977.382200] env[64020]: DEBUG oslo_vmware.api [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407962, 'name': PowerOffVM_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 977.444668] env[64020]: DEBUG nova.network.neutron [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 977.445174] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg c58f593754df4126acbab2866bca9cef in queue reply_57893177120949e6a93cb88e15cd42b4 [ 977.454661] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c58f593754df4126acbab2866bca9cef [ 977.459483] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0a90133-bd98-4bd9-91f8-47f9cfea2430 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 977.466726] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b0daa9c-8d30-40ea-9003-5c24f2fe7f94 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 977.497782] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff993524-12d3-444f-95ed-70a350662a81 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 977.504855] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-460e39de-280e-4dfa-adf2-ea5952ad9ae1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 977.519041] env[64020]: DEBUG nova.compute.provider_tree [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 977.519627] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg e0ae5cf799fa49b8b541e97f62dd6a37 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 977.528282] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e0ae5cf799fa49b8b541e97f62dd6a37 [ 977.882181] env[64020]: DEBUG oslo_vmware.api [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407962, 'name': PowerOffVM_Task, 'duration_secs': 0.122903} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 977.882497] env[64020]: DEBUG nova.virt.vmwareapi.vm_util [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Powered off the VM {{(pid=64020) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 977.882635] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Unregistering the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 977.882877] env[64020]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-af037f7c-0e98-4d78-bc8e-bb03631811da {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 977.915673] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Unregistered the VM {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 977.915900] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Deleting contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 977.916102] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Deleting the datastore file [datastore1] 77801ae9-89db-4dc7-af03-0646af73b121 {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 977.916355] env[64020]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-b7392b6e-686b-4af1-87fa-66f01a3f9947 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 977.922335] env[64020]: DEBUG oslo_vmware.api [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for the task: (returnval){ [ 977.922335] env[64020]: value = "task-407964" [ 977.922335] env[64020]: _type = "Task" [ 977.922335] env[64020]: } to complete. {{(pid=64020) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 977.929814] env[64020]: DEBUG oslo_vmware.api [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407964, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 977.947431] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-a2cd396e-bd03-4c51-8ec6-cd24d7541103" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 977.947639] env[64020]: DEBUG nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 977.947811] env[64020]: DEBUG nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 977.947969] env[64020]: DEBUG nova.network.neutron [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 977.961590] env[64020]: DEBUG nova.network.neutron [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 977.962136] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg af45d0a1a2fd4c3fa39211256c8610fd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 977.969373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af45d0a1a2fd4c3fa39211256c8610fd [ 978.023425] env[64020]: DEBUG nova.scheduler.client.report [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 978.024959] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 14bd79c5d78c47bc9d1d76209af53fd9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 978.040513] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14bd79c5d78c47bc9d1d76209af53fd9 [ 978.432132] env[64020]: DEBUG oslo_vmware.api [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Task: {'id': task-407964, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.097856} completed successfully. {{(pid=64020) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 978.432432] env[64020]: DEBUG nova.virt.vmwareapi.ds_util [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Deleted the datastore file {{(pid=64020) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 978.432617] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Deleted contents of the VM from datastore datastore1 {{(pid=64020) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 978.432790] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 978.432958] env[64020]: INFO nova.compute.manager [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Took 1.08 seconds to destroy the instance on the hypervisor. [ 978.433195] env[64020]: DEBUG oslo.service.loopingcall [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 978.433378] env[64020]: DEBUG nova.compute.manager [-] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 978.433471] env[64020]: DEBUG nova.network.neutron [-] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 978.445195] env[64020]: DEBUG nova.network.neutron [-] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 978.445623] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a385b1ccd9284c30985a4b8f7a40ef42 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 978.452947] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a385b1ccd9284c30985a4b8f7a40ef42 [ 978.465310] env[64020]: DEBUG nova.network.neutron [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 978.465765] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 4696a619bf954874804ddd66dead6066 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 978.473371] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4696a619bf954874804ddd66dead6066 [ 978.527174] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.737s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 978.527799] env[64020]: ERROR nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3e907ae1-99ed-4439-863b-3142d5834117, please check neutron logs for more information. [ 978.527799] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Traceback (most recent call last): [ 978.527799] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 978.527799] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] self.driver.spawn(context, instance, image_meta, [ 978.527799] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 978.527799] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 978.527799] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 978.527799] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] vm_ref = self.build_virtual_machine(instance, [ 978.527799] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 978.527799] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] vif_infos = vmwarevif.get_vif_info(self._session, [ 978.527799] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] for vif in network_info: [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] return self._sync_wrapper(fn, *args, **kwargs) [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] self.wait() [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] self[:] = self._gt.wait() [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] return self._exit_event.wait() [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] current.throw(*self._exc) [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 978.528165] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] result = function(*args, **kwargs) [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] return func(*args, **kwargs) [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] raise e [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] nwinfo = self.network_api.allocate_for_instance( [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] created_port_ids = self._update_ports_for_instance( [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] with excutils.save_and_reraise_exception(): [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] self.force_reraise() [ 978.528626] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 978.529006] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] raise self.value [ 978.529006] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 978.529006] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] updated_port = self._update_port( [ 978.529006] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 978.529006] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] _ensure_no_port_binding_failure(port) [ 978.529006] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 978.529006] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] raise exception.PortBindingFailed(port_id=port['id']) [ 978.529006] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] nova.exception.PortBindingFailed: Binding failed for port 3e907ae1-99ed-4439-863b-3142d5834117, please check neutron logs for more information. [ 978.529006] env[64020]: ERROR nova.compute.manager [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] [ 978.529006] env[64020]: DEBUG nova.compute.utils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Binding failed for port 3e907ae1-99ed-4439-863b-3142d5834117, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 978.529626] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.303s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 978.531470] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 500363053fec4e07a9c54d4a318f2f36 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 978.532513] env[64020]: DEBUG nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Build of instance 5f485177-eb28-417d-a74c-0e0f30ea7ce2 was re-scheduled: Binding failed for port 3e907ae1-99ed-4439-863b-3142d5834117, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 978.532907] env[64020]: DEBUG nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 978.533112] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquiring lock "refresh_cache-5f485177-eb28-417d-a74c-0e0f30ea7ce2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 978.533251] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Acquired lock "refresh_cache-5f485177-eb28-417d-a74c-0e0f30ea7ce2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 978.533397] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 978.533783] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg cbf6e1bddb8449c699d2976b0ff838ae in queue reply_57893177120949e6a93cb88e15cd42b4 [ 978.542065] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cbf6e1bddb8449c699d2976b0ff838ae [ 978.563307] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 500363053fec4e07a9c54d4a318f2f36 [ 978.790457] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "bd872d96-d664-4ecb-ad59-516d95ad0cb7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 978.790698] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "bd872d96-d664-4ecb-ad59-516d95ad0cb7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 978.791139] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg c837f4cd3dad44dcb9a1aa0321768684 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 978.800633] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c837f4cd3dad44dcb9a1aa0321768684 [ 978.947765] env[64020]: DEBUG nova.network.neutron [-] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 978.948166] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a78b866809b94759bb26c3121cd6f7ea in queue reply_57893177120949e6a93cb88e15cd42b4 [ 978.963304] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a78b866809b94759bb26c3121cd6f7ea [ 978.967465] env[64020]: INFO nova.compute.manager [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2cd396e-bd03-4c51-8ec6-cd24d7541103] Took 1.02 seconds to deallocate network for instance. [ 978.969231] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 6198917dd889497a9802af5848383f2a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 979.008412] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6198917dd889497a9802af5848383f2a [ 979.057910] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 979.143852] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 979.144428] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 029ba4cb4ad841aeb30199acea202c3d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 979.152898] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 029ba4cb4ad841aeb30199acea202c3d [ 979.167752] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f104a50-c65a-4a8d-908a-96247ba7a414 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.178161] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b24014b7-d9af-42ce-a60e-d0afb26f520b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.208410] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c20ca048-eee5-422e-9ca0-7defa68206fa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.215176] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5d9d31d-c104-4948-98f3-41a24f4fb02a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.228053] env[64020]: DEBUG nova.compute.provider_tree [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 979.228493] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 3810c0af7df34eaeb0600ef0fd9a73f3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 979.237239] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3810c0af7df34eaeb0600ef0fd9a73f3 [ 979.295774] env[64020]: DEBUG nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 979.295774] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg aa23913377e34328b48a0624be979ca6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 979.329111] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa23913377e34328b48a0624be979ca6 [ 979.455136] env[64020]: INFO nova.compute.manager [-] [instance: 77801ae9-89db-4dc7-af03-0646af73b121] Took 1.02 seconds to deallocate network for instance. [ 979.459647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg b35b4465e820460cb8a9cd93f4c70cd6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 979.475309] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 0b30dc6aeb6443faba0aa9287209face in queue reply_57893177120949e6a93cb88e15cd42b4 [ 979.488024] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b35b4465e820460cb8a9cd93f4c70cd6 [ 979.502061] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b30dc6aeb6443faba0aa9287209face [ 979.646524] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Releasing lock "refresh_cache-5f485177-eb28-417d-a74c-0e0f30ea7ce2" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 979.646765] env[64020]: DEBUG nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 979.646938] env[64020]: DEBUG nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 979.647100] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 979.661730] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 979.662264] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 3c93236191a8450180d9568210716894 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 979.672460] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c93236191a8450180d9568210716894 [ 979.731512] env[64020]: DEBUG nova.scheduler.client.report [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 979.733825] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg a729aef6163a4cd487d2ec5ea90b5ebe in queue reply_57893177120949e6a93cb88e15cd42b4 [ 979.744987] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a729aef6163a4cd487d2ec5ea90b5ebe [ 979.812714] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 979.962944] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 979.993873] env[64020]: INFO nova.scheduler.client.report [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Deleted allocations for instance a2cd396e-bd03-4c51-8ec6-cd24d7541103 [ 980.001501] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 90161fa147d945228c328206884353db in queue reply_57893177120949e6a93cb88e15cd42b4 [ 980.015851] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90161fa147d945228c328206884353db [ 980.164737] env[64020]: DEBUG nova.network.neutron [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 980.165249] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 50ff63a90ff34c81bb5551aca297725a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 980.173607] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50ff63a90ff34c81bb5551aca297725a [ 980.236032] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.706s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 980.236681] env[64020]: ERROR nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8, please check neutron logs for more information. [ 980.236681] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Traceback (most recent call last): [ 980.236681] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 980.236681] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] self.driver.spawn(context, instance, image_meta, [ 980.236681] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 980.236681] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 980.236681] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 980.236681] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] vm_ref = self.build_virtual_machine(instance, [ 980.236681] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 980.236681] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] vif_infos = vmwarevif.get_vif_info(self._session, [ 980.236681] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] for vif in network_info: [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] return self._sync_wrapper(fn, *args, **kwargs) [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] self.wait() [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] self[:] = self._gt.wait() [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] return self._exit_event.wait() [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] current.throw(*self._exc) [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 980.236990] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] result = function(*args, **kwargs) [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] return func(*args, **kwargs) [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] raise e [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] nwinfo = self.network_api.allocate_for_instance( [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] created_port_ids = self._update_ports_for_instance( [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] with excutils.save_and_reraise_exception(): [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] self.force_reraise() [ 980.237313] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 980.237647] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] raise self.value [ 980.237647] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 980.237647] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] updated_port = self._update_port( [ 980.237647] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 980.237647] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] _ensure_no_port_binding_failure(port) [ 980.237647] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 980.237647] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] raise exception.PortBindingFailed(port_id=port['id']) [ 980.237647] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] nova.exception.PortBindingFailed: Binding failed for port d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8, please check neutron logs for more information. [ 980.237647] env[64020]: ERROR nova.compute.manager [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] [ 980.237647] env[64020]: DEBUG nova.compute.utils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Binding failed for port d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 980.238569] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.730s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 980.240478] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg fb1240ffb3aa46e5a67121ba127eed34 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 980.241890] env[64020]: DEBUG nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Build of instance c5e780e2-8ef9-461e-bca2-f9d0039ce3c3 was re-scheduled: Binding failed for port d3ca9e9d-4ad9-404e-a362-a1a0fa9d98a8, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 980.242443] env[64020]: DEBUG nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 980.242643] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquiring lock "refresh_cache-c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 980.242793] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Acquired lock "refresh_cache-c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 980.242948] env[64020]: DEBUG nova.network.neutron [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 980.243306] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg dbf18835561f40e9950fbf9e45c4f6ac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 980.251577] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dbf18835561f40e9950fbf9e45c4f6ac [ 980.276913] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb1240ffb3aa46e5a67121ba127eed34 [ 980.503788] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f0b57573-7bb3-4173-b12d-ecd5370d5a7b tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "a2cd396e-bd03-4c51-8ec6-cd24d7541103" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 100.016s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 980.669277] env[64020]: INFO nova.compute.manager [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] [instance: 5f485177-eb28-417d-a74c-0e0f30ea7ce2] Took 1.02 seconds to deallocate network for instance. [ 980.670348] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 1d092c9be3334e2480d6b5ed63d004e1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 980.708014] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d092c9be3334e2480d6b5ed63d004e1 [ 980.763641] env[64020]: DEBUG nova.network.neutron [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 980.849087] env[64020]: DEBUG nova.network.neutron [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 980.849621] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 19a7bb64c4db4ae093ceefbf4dbf7aaa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 980.858569] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19a7bb64c4db4ae093ceefbf4dbf7aaa [ 980.875799] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-819f1064-9948-41a3-ad8e-d91fbf69eb0c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.883974] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3c14fbe-b5f4-4184-95b2-f1b3cb71c099 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.917507] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-626e0b00-697f-4789-b58f-cd6b7166094a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.924754] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48ef0f9c-167c-4c5a-94f0-b0b322cc1b3f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.937610] env[64020]: DEBUG nova.compute.provider_tree [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 980.938091] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg a581046f40b047a5a880ded35c63fb6b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 980.946981] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a581046f40b047a5a880ded35c63fb6b [ 981.175310] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg 8436174da8254bbcb8dd9fc1c8e1a553 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 981.206457] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8436174da8254bbcb8dd9fc1c8e1a553 [ 981.351744] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Releasing lock "refresh_cache-c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 981.351879] env[64020]: DEBUG nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 981.352070] env[64020]: DEBUG nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 981.352238] env[64020]: DEBUG nova.network.neutron [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 981.367758] env[64020]: DEBUG nova.network.neutron [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 981.368376] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg e02dd4288c8348589477aa8a952fe031 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 981.375364] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e02dd4288c8348589477aa8a952fe031 [ 981.440398] env[64020]: DEBUG nova.scheduler.client.report [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 981.442809] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 53905af96b7647db8612dbb0be9e3a90 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 981.453738] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 53905af96b7647db8612dbb0be9e3a90 [ 981.697130] env[64020]: INFO nova.scheduler.client.report [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Deleted allocations for instance 5f485177-eb28-417d-a74c-0e0f30ea7ce2 [ 981.703782] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Expecting reply to msg c5fb6c32fde1479f91442325f19b7120 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 981.720939] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5fb6c32fde1479f91442325f19b7120 [ 981.749014] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "6db545fe-881e-4c9a-b514-bd98d7528674" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 981.749296] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "6db545fe-881e-4c9a-b514-bd98d7528674" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 981.749762] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 2efe88a2b53a443da150d410044a31f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 981.768541] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2efe88a2b53a443da150d410044a31f5 [ 981.870228] env[64020]: DEBUG nova.network.neutron [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 981.870755] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 8da57e1dbe2c47f794e2b58a4904d0ef in queue reply_57893177120949e6a93cb88e15cd42b4 [ 981.878140] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8da57e1dbe2c47f794e2b58a4904d0ef [ 981.945085] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.706s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 981.945437] env[64020]: ERROR nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 65b815c2-3af4-4a8f-b05d-74efb798ffeb, please check neutron logs for more information. [ 981.945437] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Traceback (most recent call last): [ 981.945437] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 981.945437] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] self.driver.spawn(context, instance, image_meta, [ 981.945437] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 981.945437] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 981.945437] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 981.945437] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] vm_ref = self.build_virtual_machine(instance, [ 981.945437] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 981.945437] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] vif_infos = vmwarevif.get_vif_info(self._session, [ 981.945437] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] for vif in network_info: [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] return self._sync_wrapper(fn, *args, **kwargs) [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] self.wait() [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] self[:] = self._gt.wait() [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] return self._exit_event.wait() [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] current.throw(*self._exc) [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 981.945759] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] result = function(*args, **kwargs) [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] return func(*args, **kwargs) [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] raise e [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] nwinfo = self.network_api.allocate_for_instance( [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] created_port_ids = self._update_ports_for_instance( [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] with excutils.save_and_reraise_exception(): [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] self.force_reraise() [ 981.946074] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 981.946395] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] raise self.value [ 981.946395] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 981.946395] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] updated_port = self._update_port( [ 981.946395] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 981.946395] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] _ensure_no_port_binding_failure(port) [ 981.946395] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 981.946395] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] raise exception.PortBindingFailed(port_id=port['id']) [ 981.946395] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] nova.exception.PortBindingFailed: Binding failed for port 65b815c2-3af4-4a8f-b05d-74efb798ffeb, please check neutron logs for more information. [ 981.946395] env[64020]: ERROR nova.compute.manager [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] [ 981.946395] env[64020]: DEBUG nova.compute.utils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Binding failed for port 65b815c2-3af4-4a8f-b05d-74efb798ffeb, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 981.947251] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.207s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 981.948744] env[64020]: INFO nova.compute.claims [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 981.950407] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg ff810932ca21450b8a253cd55928a430 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 981.951722] env[64020]: DEBUG nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Build of instance 3ad11bab-4531-486f-88c0-3ef7153ea0f5 was re-scheduled: Binding failed for port 65b815c2-3af4-4a8f-b05d-74efb798ffeb, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 981.952121] env[64020]: DEBUG nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 981.952381] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquiring lock "refresh_cache-3ad11bab-4531-486f-88c0-3ef7153ea0f5" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 981.952479] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Acquired lock "refresh_cache-3ad11bab-4531-486f-88c0-3ef7153ea0f5" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 981.952635] env[64020]: DEBUG nova.network.neutron [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 981.952981] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg d209173a5b2b44c5aaab278ecc94c927 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 981.958367] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d209173a5b2b44c5aaab278ecc94c927 [ 981.980990] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff810932ca21450b8a253cd55928a430 [ 982.205774] env[64020]: DEBUG oslo_concurrency.lockutils [None req-c66c9626-6488-4c96-a900-991e27a388e0 tempest-ServersTestMultiNic-401204283 tempest-ServersTestMultiNic-401204283-project-member] Lock "5f485177-eb28-417d-a74c-0e0f30ea7ce2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 101.696s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 982.252032] env[64020]: DEBUG nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 982.253768] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 15b7baf5323544a5ad3343a9e97d6706 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 982.282355] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 15b7baf5323544a5ad3343a9e97d6706 [ 982.372878] env[64020]: INFO nova.compute.manager [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] [instance: c5e780e2-8ef9-461e-bca2-f9d0039ce3c3] Took 1.02 seconds to deallocate network for instance. [ 982.374845] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 7fa7b133b7ac49588bab8d474175c733 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 982.417765] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7fa7b133b7ac49588bab8d474175c733 [ 982.456997] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 68af0feeabf846e790383c14c611291c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 982.465292] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 68af0feeabf846e790383c14c611291c [ 982.474117] env[64020]: DEBUG nova.network.neutron [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 982.551780] env[64020]: DEBUG nova.network.neutron [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 982.552366] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 65da019082e94dedaddea5111e3bef57 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 982.567527] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65da019082e94dedaddea5111e3bef57 [ 982.773070] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 982.879072] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg cb8e30a6675443fabcfac349ea7af9a8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 982.910886] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cb8e30a6675443fabcfac349ea7af9a8 [ 983.055394] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Releasing lock "refresh_cache-3ad11bab-4531-486f-88c0-3ef7153ea0f5" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 983.055895] env[64020]: DEBUG nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 983.056226] env[64020]: DEBUG nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 983.056554] env[64020]: DEBUG nova.network.neutron [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 983.079496] env[64020]: DEBUG nova.network.neutron [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 983.080422] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 9000c1ff904f4659bc2a26a5758031a8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 983.087162] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9000c1ff904f4659bc2a26a5758031a8 [ 983.112950] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6a8f3bd-2425-4cf6-bc44-49ec5eaff6e2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 983.124612] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6c09714-1ed3-4485-aa5c-cba0778080a0 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 983.155033] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7669bcb0-bcc2-4bb6-bce0-b66b56311ede {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 983.162868] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3c6c371-819c-418a-8f9c-b49fa69b6c85 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 983.176444] env[64020]: DEBUG nova.compute.provider_tree [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 983.176942] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 9ee3083bf1cc4ddf9fe6b8f0362f2af7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 983.185193] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ee3083bf1cc4ddf9fe6b8f0362f2af7 [ 983.412857] env[64020]: INFO nova.scheduler.client.report [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Deleted allocations for instance c5e780e2-8ef9-461e-bca2-f9d0039ce3c3 [ 983.419580] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Expecting reply to msg 26cfe05fb2d74f2fb2b43b6c883d8a8a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 983.433589] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26cfe05fb2d74f2fb2b43b6c883d8a8a [ 983.582258] env[64020]: DEBUG nova.network.neutron [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 983.582804] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg c5e66a8f1fd442a5896b6401d96e6fc9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 983.590978] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5e66a8f1fd442a5896b6401d96e6fc9 [ 983.680394] env[64020]: DEBUG nova.scheduler.client.report [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 983.684076] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg c0cd71fc7c334bab92f629b9327b5669 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 983.697624] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0cd71fc7c334bab92f629b9327b5669 [ 983.922012] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4157fb4d-3b25-4e22-b6be-f44b7a385d5d tempest-ImagesTestJSON-1663560030 tempest-ImagesTestJSON-1663560030-project-member] Lock "c5e780e2-8ef9-461e-bca2-f9d0039ce3c3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 97.349s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 984.085573] env[64020]: INFO nova.compute.manager [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] [instance: 3ad11bab-4531-486f-88c0-3ef7153ea0f5] Took 1.03 seconds to deallocate network for instance. [ 984.087455] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 3cdec1edaf074b88a747601b4717e318 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 984.128279] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3cdec1edaf074b88a747601b4717e318 [ 984.187046] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.239s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 984.187046] env[64020]: DEBUG nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 984.188601] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg dda21a683ff74af7b54884bf1265db41 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 984.189602] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.949s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 984.191949] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg f2ea383a67ca4246b02c285cbd8a6ce0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 984.219131] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dda21a683ff74af7b54884bf1265db41 [ 984.233626] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f2ea383a67ca4246b02c285cbd8a6ce0 [ 984.592479] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg c13b1a260d6d445cb288e620c70b6b94 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 984.628946] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c13b1a260d6d445cb288e620c70b6b94 [ 984.695467] env[64020]: DEBUG nova.compute.utils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 984.695890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 66bfcb556c0f4f20aec50c5105f51b95 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 984.697471] env[64020]: DEBUG nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 984.697643] env[64020]: DEBUG nova.network.neutron [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 984.711636] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66bfcb556c0f4f20aec50c5105f51b95 [ 984.778322] env[64020]: DEBUG nova.policy [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fd08d772c4dd4bf5986d37b8474d30a1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '540a31f288b54221a4d0348b147e40e7', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 984.888155] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f73ee9f-38c2-4f70-85ec-7ab6ed42e00f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.896254] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-818a14c4-ba51-474a-bdee-b5f5dfa50cd5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.931955] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba50427f-6b32-40ea-afc1-c004ed52641e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.942672] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d245a1a9-0d95-4dee-994d-bda717368924 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.960607] env[64020]: DEBUG nova.compute.provider_tree [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 984.961563] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg dfecb5cbe62e4aab82546db7f7f348f4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 984.969177] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dfecb5cbe62e4aab82546db7f7f348f4 [ 985.117930] env[64020]: INFO nova.scheduler.client.report [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Deleted allocations for instance 3ad11bab-4531-486f-88c0-3ef7153ea0f5 [ 985.123581] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Expecting reply to msg 31dd1a82a114401b9967bf2140ed712e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 985.142298] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 31dd1a82a114401b9967bf2140ed712e [ 985.203633] env[64020]: DEBUG nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 985.205336] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg fe9bd9022a70410fbcda8bae1b7c43a9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 985.248296] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe9bd9022a70410fbcda8bae1b7c43a9 [ 985.270645] env[64020]: DEBUG nova.network.neutron [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Successfully created port: b23be879-2ee0-4d32-ba16-5aa71dcf946f {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 985.464619] env[64020]: DEBUG nova.scheduler.client.report [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 985.467107] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg ebb8d811d5114597ab0fed8ac22f624e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 985.481302] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ebb8d811d5114597ab0fed8ac22f624e [ 985.625963] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a8ac9691-43ac-4cee-83df-705687094b92 tempest-AttachVolumeTestJSON-537045632 tempest-AttachVolumeTestJSON-537045632-project-member] Lock "3ad11bab-4531-486f-88c0-3ef7153ea0f5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 98.747s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 985.710041] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 396d625426e04bb3b3fd28e94cef6e63 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 985.741806] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 396d625426e04bb3b3fd28e94cef6e63 [ 985.969533] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.780s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 985.970166] env[64020]: ERROR nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port bc5e06b8-8c09-40b0-852c-2fb750a365c1, please check neutron logs for more information. [ 985.970166] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Traceback (most recent call last): [ 985.970166] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 985.970166] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] self.driver.spawn(context, instance, image_meta, [ 985.970166] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 985.970166] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 985.970166] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 985.970166] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] vm_ref = self.build_virtual_machine(instance, [ 985.970166] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 985.970166] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] vif_infos = vmwarevif.get_vif_info(self._session, [ 985.970166] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] for vif in network_info: [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] return self._sync_wrapper(fn, *args, **kwargs) [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] self.wait() [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] self[:] = self._gt.wait() [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] return self._exit_event.wait() [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] current.throw(*self._exc) [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 985.970643] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] result = function(*args, **kwargs) [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] return func(*args, **kwargs) [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] raise e [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] nwinfo = self.network_api.allocate_for_instance( [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] created_port_ids = self._update_ports_for_instance( [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] with excutils.save_and_reraise_exception(): [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] self.force_reraise() [ 985.971128] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 985.971576] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] raise self.value [ 985.971576] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 985.971576] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] updated_port = self._update_port( [ 985.971576] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 985.971576] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] _ensure_no_port_binding_failure(port) [ 985.971576] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 985.971576] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] raise exception.PortBindingFailed(port_id=port['id']) [ 985.971576] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] nova.exception.PortBindingFailed: Binding failed for port bc5e06b8-8c09-40b0-852c-2fb750a365c1, please check neutron logs for more information. [ 985.971576] env[64020]: ERROR nova.compute.manager [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] [ 985.971576] env[64020]: DEBUG nova.compute.utils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Binding failed for port bc5e06b8-8c09-40b0-852c-2fb750a365c1, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 985.972559] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.657s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 985.974573] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 904b11d7d8e54440bcc2af173b5e36bf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 985.976188] env[64020]: DEBUG nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Build of instance 50e9cb9c-10fd-466d-9b11-5175d7955ac9 was re-scheduled: Binding failed for port bc5e06b8-8c09-40b0-852c-2fb750a365c1, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 985.976589] env[64020]: DEBUG nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 985.976808] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "refresh_cache-50e9cb9c-10fd-466d-9b11-5175d7955ac9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 985.976950] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquired lock "refresh_cache-50e9cb9c-10fd-466d-9b11-5175d7955ac9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 985.977106] env[64020]: DEBUG nova.network.neutron [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 985.977461] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg de8937e4ef5a4b61a3dba1fe3b233313 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 985.994168] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de8937e4ef5a4b61a3dba1fe3b233313 [ 986.014791] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 904b11d7d8e54440bcc2af173b5e36bf [ 986.212797] env[64020]: DEBUG nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 986.236418] env[64020]: DEBUG nova.virt.hardware [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 986.236669] env[64020]: DEBUG nova.virt.hardware [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 986.236821] env[64020]: DEBUG nova.virt.hardware [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 986.238486] env[64020]: DEBUG nova.virt.hardware [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 986.238486] env[64020]: DEBUG nova.virt.hardware [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 986.238486] env[64020]: DEBUG nova.virt.hardware [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 986.238486] env[64020]: DEBUG nova.virt.hardware [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 986.238486] env[64020]: DEBUG nova.virt.hardware [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 986.238729] env[64020]: DEBUG nova.virt.hardware [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 986.238729] env[64020]: DEBUG nova.virt.hardware [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 986.238729] env[64020]: DEBUG nova.virt.hardware [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 986.238936] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-040b7efc-14de-4eb9-832d-ab7166853d85 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.247356] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2274a8fc-408c-437f-bdf5-907f87320d97 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.503287] env[64020]: DEBUG nova.network.neutron [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 986.605936] env[64020]: DEBUG nova.compute.manager [req-b35a0ea0-f15a-47e9-8c2e-2fa018e32dd0 req-e3a902e9-5e4d-4739-8e2b-c512e940a15a service nova] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Received event network-changed-b23be879-2ee0-4d32-ba16-5aa71dcf946f {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 986.606130] env[64020]: DEBUG nova.compute.manager [req-b35a0ea0-f15a-47e9-8c2e-2fa018e32dd0 req-e3a902e9-5e4d-4739-8e2b-c512e940a15a service nova] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Refreshing instance network info cache due to event network-changed-b23be879-2ee0-4d32-ba16-5aa71dcf946f. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 986.606434] env[64020]: DEBUG oslo_concurrency.lockutils [req-b35a0ea0-f15a-47e9-8c2e-2fa018e32dd0 req-e3a902e9-5e4d-4739-8e2b-c512e940a15a service nova] Acquiring lock "refresh_cache-3c6ddf08-d0f3-444b-8249-f9dabeeef87f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 986.606537] env[64020]: DEBUG oslo_concurrency.lockutils [req-b35a0ea0-f15a-47e9-8c2e-2fa018e32dd0 req-e3a902e9-5e4d-4739-8e2b-c512e940a15a service nova] Acquired lock "refresh_cache-3c6ddf08-d0f3-444b-8249-f9dabeeef87f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 986.606620] env[64020]: DEBUG nova.network.neutron [req-b35a0ea0-f15a-47e9-8c2e-2fa018e32dd0 req-e3a902e9-5e4d-4739-8e2b-c512e940a15a service nova] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Refreshing network info cache for port b23be879-2ee0-4d32-ba16-5aa71dcf946f {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 986.607041] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-b35a0ea0-f15a-47e9-8c2e-2fa018e32dd0 req-e3a902e9-5e4d-4739-8e2b-c512e940a15a service nova] Expecting reply to msg 462a864df6eb4380b0a9ecb481ffc291 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 986.609837] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6ded82b-7954-4b10-b753-7985bf430b2c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.620449] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 462a864df6eb4380b0a9ecb481ffc291 [ 986.626835] env[64020]: DEBUG nova.network.neutron [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 986.627339] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg de7f45b12eb6476693413c35b20f7e74 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 986.628893] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fae10115-e3ec-4154-8baf-abb94a36f0bc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.663749] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de7f45b12eb6476693413c35b20f7e74 [ 986.664830] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e8295e5-3c6a-4020-8be0-9f775ff3c3a3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.672723] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5544be94-27a6-478c-bc48-e9676bb0d3c7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.687893] env[64020]: DEBUG nova.compute.provider_tree [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 986.688748] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg bf53a078b0214d08b65b1cc3a5e487c7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 986.696344] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf53a078b0214d08b65b1cc3a5e487c7 [ 986.699517] env[64020]: DEBUG nova.network.neutron [req-b35a0ea0-f15a-47e9-8c2e-2fa018e32dd0 req-e3a902e9-5e4d-4739-8e2b-c512e940a15a service nova] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 986.869531] env[64020]: DEBUG nova.network.neutron [req-b35a0ea0-f15a-47e9-8c2e-2fa018e32dd0 req-e3a902e9-5e4d-4739-8e2b-c512e940a15a service nova] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 986.870126] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-b35a0ea0-f15a-47e9-8c2e-2fa018e32dd0 req-e3a902e9-5e4d-4739-8e2b-c512e940a15a service nova] Expecting reply to msg 8991a67a0dce441f9e73f3f53efd51c5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 986.879055] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8991a67a0dce441f9e73f3f53efd51c5 [ 986.924023] env[64020]: ERROR nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b23be879-2ee0-4d32-ba16-5aa71dcf946f, please check neutron logs for more information. [ 986.924023] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 986.924023] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 986.924023] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 986.924023] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 986.924023] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 986.924023] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 986.924023] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 986.924023] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 986.924023] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 986.924023] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 986.924023] env[64020]: ERROR nova.compute.manager raise self.value [ 986.924023] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 986.924023] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 986.924023] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 986.924023] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 986.924580] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 986.924580] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 986.924580] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b23be879-2ee0-4d32-ba16-5aa71dcf946f, please check neutron logs for more information. [ 986.924580] env[64020]: ERROR nova.compute.manager [ 986.924580] env[64020]: Traceback (most recent call last): [ 986.924580] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 986.924580] env[64020]: listener.cb(fileno) [ 986.924580] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 986.924580] env[64020]: result = function(*args, **kwargs) [ 986.924580] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 986.924580] env[64020]: return func(*args, **kwargs) [ 986.924580] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 986.924580] env[64020]: raise e [ 986.924580] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 986.924580] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 986.924580] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 986.924580] env[64020]: created_port_ids = self._update_ports_for_instance( [ 986.924580] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 986.924580] env[64020]: with excutils.save_and_reraise_exception(): [ 986.924580] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 986.924580] env[64020]: self.force_reraise() [ 986.924580] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 986.924580] env[64020]: raise self.value [ 986.924580] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 986.924580] env[64020]: updated_port = self._update_port( [ 986.924580] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 986.924580] env[64020]: _ensure_no_port_binding_failure(port) [ 986.924580] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 986.924580] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 986.925483] env[64020]: nova.exception.PortBindingFailed: Binding failed for port b23be879-2ee0-4d32-ba16-5aa71dcf946f, please check neutron logs for more information. [ 986.925483] env[64020]: Removing descriptor: 16 [ 986.925483] env[64020]: ERROR nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b23be879-2ee0-4d32-ba16-5aa71dcf946f, please check neutron logs for more information. [ 986.925483] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Traceback (most recent call last): [ 986.925483] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 986.925483] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] yield resources [ 986.925483] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 986.925483] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] self.driver.spawn(context, instance, image_meta, [ 986.925483] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 986.925483] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 986.925483] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 986.925483] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] vm_ref = self.build_virtual_machine(instance, [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] vif_infos = vmwarevif.get_vif_info(self._session, [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] for vif in network_info: [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] return self._sync_wrapper(fn, *args, **kwargs) [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] self.wait() [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] self[:] = self._gt.wait() [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] return self._exit_event.wait() [ 986.925863] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] result = hub.switch() [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] return self.greenlet.switch() [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] result = function(*args, **kwargs) [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] return func(*args, **kwargs) [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] raise e [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] nwinfo = self.network_api.allocate_for_instance( [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 986.926340] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] created_port_ids = self._update_ports_for_instance( [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] with excutils.save_and_reraise_exception(): [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] self.force_reraise() [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] raise self.value [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] updated_port = self._update_port( [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] _ensure_no_port_binding_failure(port) [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 986.926741] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] raise exception.PortBindingFailed(port_id=port['id']) [ 986.927110] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] nova.exception.PortBindingFailed: Binding failed for port b23be879-2ee0-4d32-ba16-5aa71dcf946f, please check neutron logs for more information. [ 986.927110] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] [ 986.927110] env[64020]: INFO nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Terminating instance [ 986.927204] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "refresh_cache-3c6ddf08-d0f3-444b-8249-f9dabeeef87f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 987.133635] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Releasing lock "refresh_cache-50e9cb9c-10fd-466d-9b11-5175d7955ac9" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 987.133912] env[64020]: DEBUG nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 987.134056] env[64020]: DEBUG nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 987.134217] env[64020]: DEBUG nova.network.neutron [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 987.151468] env[64020]: DEBUG nova.network.neutron [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 987.152029] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 0c9657a04ecb416da561a89477998387 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 987.160983] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c9657a04ecb416da561a89477998387 [ 987.191934] env[64020]: DEBUG nova.scheduler.client.report [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 987.194568] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg b743912793e741698ae0f66ea66eb624 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 987.212736] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b743912793e741698ae0f66ea66eb624 [ 987.372683] env[64020]: DEBUG oslo_concurrency.lockutils [req-b35a0ea0-f15a-47e9-8c2e-2fa018e32dd0 req-e3a902e9-5e4d-4739-8e2b-c512e940a15a service nova] Releasing lock "refresh_cache-3c6ddf08-d0f3-444b-8249-f9dabeeef87f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 987.373172] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquired lock "refresh_cache-3c6ddf08-d0f3-444b-8249-f9dabeeef87f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 987.373421] env[64020]: DEBUG nova.network.neutron [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 987.373899] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 23dc8be2ddb34749878fa95702d2c7b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 987.384841] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 23dc8be2ddb34749878fa95702d2c7b3 [ 987.654233] env[64020]: DEBUG nova.network.neutron [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 987.654775] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 64f9d10460044e4ca471aa5b6a8b1399 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 987.663109] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64f9d10460044e4ca471aa5b6a8b1399 [ 987.697669] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.724s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 987.697859] env[64020]: ERROR nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port af0a735b-fb31-43c7-a943-6d3dcdfa83c4, please check neutron logs for more information. [ 987.697859] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Traceback (most recent call last): [ 987.697859] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 987.697859] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] self.driver.spawn(context, instance, image_meta, [ 987.697859] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 987.697859] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 987.697859] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 987.697859] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] vm_ref = self.build_virtual_machine(instance, [ 987.697859] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 987.697859] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] vif_infos = vmwarevif.get_vif_info(self._session, [ 987.697859] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] for vif in network_info: [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] return self._sync_wrapper(fn, *args, **kwargs) [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] self.wait() [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] self[:] = self._gt.wait() [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] return self._exit_event.wait() [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] result = hub.switch() [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 987.698160] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] return self.greenlet.switch() [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] result = function(*args, **kwargs) [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] return func(*args, **kwargs) [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] raise e [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] nwinfo = self.network_api.allocate_for_instance( [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] created_port_ids = self._update_ports_for_instance( [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] with excutils.save_and_reraise_exception(): [ 987.698545] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 987.698890] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] self.force_reraise() [ 987.698890] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 987.698890] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] raise self.value [ 987.698890] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 987.698890] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] updated_port = self._update_port( [ 987.698890] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 987.698890] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] _ensure_no_port_binding_failure(port) [ 987.698890] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 987.698890] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] raise exception.PortBindingFailed(port_id=port['id']) [ 987.698890] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] nova.exception.PortBindingFailed: Binding failed for port af0a735b-fb31-43c7-a943-6d3dcdfa83c4, please check neutron logs for more information. [ 987.698890] env[64020]: ERROR nova.compute.manager [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] [ 987.699260] env[64020]: DEBUG nova.compute.utils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Binding failed for port af0a735b-fb31-43c7-a943-6d3dcdfa83c4, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 987.700297] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.666s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 987.702227] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 25aa4bc38122485fa12ca917f490589a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 987.703835] env[64020]: DEBUG nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Build of instance bf317705-3bf8-4522-b41a-fc023bf766d0 was re-scheduled: Binding failed for port af0a735b-fb31-43c7-a943-6d3dcdfa83c4, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 987.704360] env[64020]: DEBUG nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 987.704523] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquiring lock "refresh_cache-bf317705-3bf8-4522-b41a-fc023bf766d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 987.704664] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Acquired lock "refresh_cache-bf317705-3bf8-4522-b41a-fc023bf766d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 987.704816] env[64020]: DEBUG nova.network.neutron [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 987.705186] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 36f3150f65e64ffa9df9e6965b4f7961 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 987.713533] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36f3150f65e64ffa9df9e6965b4f7961 [ 987.746602] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 25aa4bc38122485fa12ca917f490589a [ 987.903868] env[64020]: DEBUG nova.network.neutron [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 988.052388] env[64020]: DEBUG nova.network.neutron [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 988.053096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg a573d1e162944bc1b3ff9bba67d21c38 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 988.063124] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a573d1e162944bc1b3ff9bba67d21c38 [ 988.156719] env[64020]: INFO nova.compute.manager [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 50e9cb9c-10fd-466d-9b11-5175d7955ac9] Took 1.02 seconds to deallocate network for instance. [ 988.158560] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 6fa47866666d4503bff6e53cdba3bbe7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 988.201725] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6fa47866666d4503bff6e53cdba3bbe7 [ 988.230799] env[64020]: DEBUG nova.network.neutron [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 988.317392] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2c69def-868f-4764-900c-7123f7be5401 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.320938] env[64020]: DEBUG nova.network.neutron [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 988.321460] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg bcb996e4c1c14f2a9e5b860fab191d0e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 988.325396] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db46de1e-4e7c-4bef-8d2b-217bedcda56a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.329620] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bcb996e4c1c14f2a9e5b860fab191d0e [ 988.358422] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1875a3a8-6f7d-4fd3-93b6-747147e9275b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.365722] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c314dff0-1ec5-4e2a-ac46-34ef98b3e981 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.379119] env[64020]: DEBUG nova.compute.provider_tree [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 988.379644] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 43e71319d8474c1fa0e6ecf4bd608c3e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 988.390653] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43e71319d8474c1fa0e6ecf4bd608c3e [ 988.555186] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Releasing lock "refresh_cache-3c6ddf08-d0f3-444b-8249-f9dabeeef87f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 988.555608] env[64020]: DEBUG nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 988.555801] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 988.556124] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ca7453d4-9d76-437d-b74b-00fd5fc9f3e6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.565817] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86875269-dae5-4db0-812a-4a729c0a9b9b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.587588] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3c6ddf08-d0f3-444b-8249-f9dabeeef87f could not be found. [ 988.587822] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 988.588000] env[64020]: INFO nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Took 0.03 seconds to destroy the instance on the hypervisor. [ 988.588286] env[64020]: DEBUG oslo.service.loopingcall [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 988.588513] env[64020]: DEBUG nova.compute.manager [-] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 988.588607] env[64020]: DEBUG nova.network.neutron [-] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 988.663851] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 92cd88e327d8443b98573eccda7e4d43 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 988.694095] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92cd88e327d8443b98573eccda7e4d43 [ 988.743039] env[64020]: DEBUG nova.network.neutron [-] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 988.743658] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3b154c1be8894717aac5230bcdfb6bd5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 988.756401] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b154c1be8894717aac5230bcdfb6bd5 [ 988.761591] env[64020]: DEBUG nova.compute.manager [req-00a5cbbe-ea80-42cf-b1f4-66cfbea67464 req-ac8a1039-1a6b-474f-abc7-eada54774e82 service nova] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Received event network-vif-deleted-b23be879-2ee0-4d32-ba16-5aa71dcf946f {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 988.825430] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Releasing lock "refresh_cache-bf317705-3bf8-4522-b41a-fc023bf766d0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 988.825657] env[64020]: DEBUG nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 988.825832] env[64020]: DEBUG nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 988.825988] env[64020]: DEBUG nova.network.neutron [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 988.846896] env[64020]: DEBUG nova.network.neutron [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 988.847427] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg cda650a2acbd48f881ca96a91dee63a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 988.854774] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cda650a2acbd48f881ca96a91dee63a2 [ 988.883147] env[64020]: DEBUG nova.scheduler.client.report [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 988.885545] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 0fa6e91487154e8bbc3d1d9eb5790bb3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 988.898347] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0fa6e91487154e8bbc3d1d9eb5790bb3 [ 989.199790] env[64020]: INFO nova.scheduler.client.report [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Deleted allocations for instance 50e9cb9c-10fd-466d-9b11-5175d7955ac9 [ 989.206083] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 95a52b954a614560ae24c56a0ab911c5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 989.218479] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95a52b954a614560ae24c56a0ab911c5 [ 989.245566] env[64020]: DEBUG nova.network.neutron [-] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 989.246033] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 36b5f57c058f4ca7bfd60ea2ddbb4b45 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 989.253235] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36b5f57c058f4ca7bfd60ea2ddbb4b45 [ 989.350077] env[64020]: DEBUG nova.network.neutron [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 989.350628] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 4efb2bfab8814618a704382a2f1b4f93 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 989.358417] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4efb2bfab8814618a704382a2f1b4f93 [ 989.388097] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.688s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 989.389026] env[64020]: ERROR nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b99f58dd-63de-41a1-853f-7ee402bf6c3c, please check neutron logs for more information. [ 989.389026] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Traceback (most recent call last): [ 989.389026] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 989.389026] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] self.driver.spawn(context, instance, image_meta, [ 989.389026] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 989.389026] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 989.389026] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 989.389026] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] vm_ref = self.build_virtual_machine(instance, [ 989.389026] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 989.389026] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] vif_infos = vmwarevif.get_vif_info(self._session, [ 989.389026] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] for vif in network_info: [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] return self._sync_wrapper(fn, *args, **kwargs) [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] self.wait() [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] self[:] = self._gt.wait() [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] return self._exit_event.wait() [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] current.throw(*self._exc) [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 989.389555] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] result = function(*args, **kwargs) [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] return func(*args, **kwargs) [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] raise e [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] nwinfo = self.network_api.allocate_for_instance( [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] created_port_ids = self._update_ports_for_instance( [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] with excutils.save_and_reraise_exception(): [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] self.force_reraise() [ 989.390160] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 989.390710] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] raise self.value [ 989.390710] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 989.390710] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] updated_port = self._update_port( [ 989.390710] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 989.390710] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] _ensure_no_port_binding_failure(port) [ 989.390710] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 989.390710] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] raise exception.PortBindingFailed(port_id=port['id']) [ 989.390710] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] nova.exception.PortBindingFailed: Binding failed for port b99f58dd-63de-41a1-853f-7ee402bf6c3c, please check neutron logs for more information. [ 989.390710] env[64020]: ERROR nova.compute.manager [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] [ 989.390710] env[64020]: DEBUG nova.compute.utils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Binding failed for port b99f58dd-63de-41a1-853f-7ee402bf6c3c, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 989.391859] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.064s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 989.393710] env[64020]: INFO nova.compute.claims [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 989.395409] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg b234c2d4a8f64da89c1b9b79f6355b66 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 989.396917] env[64020]: DEBUG nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Build of instance 9d199d06-86cb-4d2f-894d-1ce0ef29235f was re-scheduled: Binding failed for port b99f58dd-63de-41a1-853f-7ee402bf6c3c, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 989.397371] env[64020]: DEBUG nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 989.397720] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Acquiring lock "refresh_cache-9d199d06-86cb-4d2f-894d-1ce0ef29235f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 989.397913] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Acquired lock "refresh_cache-9d199d06-86cb-4d2f-894d-1ce0ef29235f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 989.398258] env[64020]: DEBUG nova.network.neutron [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 989.398656] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg f29cb6b956824d469a386e365735fc2e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 989.405193] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f29cb6b956824d469a386e365735fc2e [ 989.431675] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b234c2d4a8f64da89c1b9b79f6355b66 [ 989.708585] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e6453d80-41e1-42f5-bcd1-cc6c2bff7a92 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "50e9cb9c-10fd-466d-9b11-5175d7955ac9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 98.674s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 989.748223] env[64020]: INFO nova.compute.manager [-] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Took 1.16 seconds to deallocate network for instance. [ 989.750881] env[64020]: DEBUG nova.compute.claims [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 989.751073] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 989.852586] env[64020]: INFO nova.compute.manager [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] [instance: bf317705-3bf8-4522-b41a-fc023bf766d0] Took 1.03 seconds to deallocate network for instance. [ 989.854505] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 3af6f806342e4bb79666b5e4fb3adbe7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 989.897597] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3af6f806342e4bb79666b5e4fb3adbe7 [ 989.902051] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 22681acac4ee46dc86fdf61e2d1dfa99 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 989.912801] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 22681acac4ee46dc86fdf61e2d1dfa99 [ 989.925458] env[64020]: DEBUG nova.network.neutron [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 990.105447] env[64020]: DEBUG nova.network.neutron [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 990.105975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 2139474bf1bc4a84985c37c3ed7724e1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 990.114841] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2139474bf1bc4a84985c37c3ed7724e1 [ 990.358648] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 4d877e2a8d074a4fb79e6ba0ce8a479a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 990.391869] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d877e2a8d074a4fb79e6ba0ce8a479a [ 990.523493] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d4ae4fef-a18d-4acc-9495-4a3a2759e1b3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 990.536066] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bcc959ad-3112-4331-a91a-bd2c8a5db73c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 990.569343] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3ed718d-00b1-4b28-973f-b92f61602cbf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 990.577510] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6081061c-4c1b-4aa1-b572-68af764c0548 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 990.593208] env[64020]: DEBUG nova.compute.provider_tree [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 990.593729] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg c499734310ae4c1b80c512eb1158de26 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 990.604932] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c499734310ae4c1b80c512eb1158de26 [ 990.608192] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Releasing lock "refresh_cache-9d199d06-86cb-4d2f-894d-1ce0ef29235f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 990.608454] env[64020]: DEBUG nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 990.608679] env[64020]: DEBUG nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 990.608887] env[64020]: DEBUG nova.network.neutron [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 990.625396] env[64020]: DEBUG nova.network.neutron [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 990.626093] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 4e6407c3b7cb4cc48f93476cd8f5d0aa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 990.632553] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e6407c3b7cb4cc48f93476cd8f5d0aa [ 990.878955] env[64020]: INFO nova.scheduler.client.report [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Deleted allocations for instance bf317705-3bf8-4522-b41a-fc023bf766d0 [ 990.885088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Expecting reply to msg 09dc5fb566d84e7d96797b6dabf4815b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 990.900672] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 09dc5fb566d84e7d96797b6dabf4815b [ 991.096343] env[64020]: DEBUG nova.scheduler.client.report [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 991.098787] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 93434ef89b4b4b67aaf4bb9997f2bfe5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 991.109755] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93434ef89b4b4b67aaf4bb9997f2bfe5 [ 991.127845] env[64020]: DEBUG nova.network.neutron [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 991.128399] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 19884176ec6f4c309a957017e0e05fa3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 991.137604] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19884176ec6f4c309a957017e0e05fa3 [ 991.387422] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dca00607-5fbb-43d8-aa96-49a30e8260f6 tempest-AttachVolumeNegativeTest-948750044 tempest-AttachVolumeNegativeTest-948750044-project-member] Lock "bf317705-3bf8-4522-b41a-fc023bf766d0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 90.429s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 991.606088] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.209s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 991.606088] env[64020]: DEBUG nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 991.606088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 2fb59691ec9c493aa6817776d08c4e5a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 991.606088] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 11.792s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 991.607206] env[64020]: INFO nova.compute.claims [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 991.609285] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg f38aacf92b4140f789ddf1b4922a7722 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 991.630465] env[64020]: INFO nova.compute.manager [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] [instance: 9d199d06-86cb-4d2f-894d-1ce0ef29235f] Took 1.02 seconds to deallocate network for instance. [ 991.633607] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 5553fdf834c54559aa8a942855445e26 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 991.655207] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f38aacf92b4140f789ddf1b4922a7722 [ 991.665168] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2fb59691ec9c493aa6817776d08c4e5a [ 991.693639] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5553fdf834c54559aa8a942855445e26 [ 991.712407] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Acquiring lock "8f90d1e0-6ede-4e89-8423-170c0d0e605f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 991.712747] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Lock "8f90d1e0-6ede-4e89-8423-170c0d0e605f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 991.713258] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg a62fc94b38ee4a3e8c50b7e27d28e151 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 991.722902] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a62fc94b38ee4a3e8c50b7e27d28e151 [ 992.112416] env[64020]: DEBUG nova.compute.utils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 992.113111] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 9c781702ab754db7a2109067a6a0b283 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 992.114817] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 7f8e7e09f6744acfbccd779a429f2ba8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 992.115582] env[64020]: DEBUG nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 992.115749] env[64020]: DEBUG nova.network.neutron [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 992.126002] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f8e7e09f6744acfbccd779a429f2ba8 [ 992.137825] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c781702ab754db7a2109067a6a0b283 [ 992.140625] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg 250b5fc03a0148259f41a0610581979f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 992.183532] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 250b5fc03a0148259f41a0610581979f [ 992.185231] env[64020]: DEBUG nova.policy [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '045421bc288e438fb9229e8ffaff0686', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cb01421fd6f24fc783dd1e06a9901b94', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 992.215635] env[64020]: DEBUG nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 992.217523] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 30745a1b9d0541d1a6d4b1caf0f1fef9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 992.260081] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30745a1b9d0541d1a6d4b1caf0f1fef9 [ 992.480172] env[64020]: DEBUG nova.network.neutron [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Successfully created port: 3cd0eb57-3579-49ff-8787-df8d6a4eadab {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 992.622351] env[64020]: DEBUG nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 992.624065] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 11103f99602a488292fc4e32d073e7bd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 992.664064] env[64020]: INFO nova.scheduler.client.report [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Deleted allocations for instance 9d199d06-86cb-4d2f-894d-1ce0ef29235f [ 992.671815] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Expecting reply to msg e7455f5ab8024f49b66b4a1c3a232f7b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 992.678855] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 11103f99602a488292fc4e32d073e7bd [ 992.695654] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e7455f5ab8024f49b66b4a1c3a232f7b [ 992.736294] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 992.763552] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e903be98-d8e2-47fd-8dbc-537894b05c91 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.771528] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6649457-8d38-4300-9a10-e8f965e12ab1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.802437] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50cc1035-99b2-4ef1-8f0c-6f70a3908ceb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.808018] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "d585fc50-c5a6-47bc-8672-498ac81c88f1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 992.808270] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "d585fc50-c5a6-47bc-8672-498ac81c88f1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 992.808738] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg aa0e48153ba84e2fae5800344aed0ee8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 992.814601] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-750c5bb5-0a8f-4c15-8b81-17c237a87a79 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.819742] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa0e48153ba84e2fae5800344aed0ee8 [ 992.829922] env[64020]: DEBUG nova.compute.provider_tree [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 992.830403] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 1f5fb663a42d4d1c9c8dcf2a1a326e3d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 992.867998] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f5fb663a42d4d1c9c8dcf2a1a326e3d [ 993.129530] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 8bceeac058b84393840e6cbc3ac2e66b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 993.168289] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8bceeac058b84393840e6cbc3ac2e66b [ 993.175106] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4b7792b3-0a3d-4dab-898d-4910bf1f9600 tempest-ServerTagsTestJSON-1822166312 tempest-ServerTagsTestJSON-1822166312-project-member] Lock "9d199d06-86cb-4d2f-894d-1ce0ef29235f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 87.260s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 993.313962] env[64020]: DEBUG nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 993.313962] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg b0468ae0ee764072859eed3bfad7fe93 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 993.340220] env[64020]: DEBUG nova.scheduler.client.report [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 993.340220] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg bdb7c1c2feae47a3a5bfdf79ff0e8c5e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 993.340220] env[64020]: DEBUG nova.compute.manager [req-712e4443-8b07-4967-8ed3-5fc9cfa9e1f6 req-93775ee9-bb32-4f15-a281-edb0795120b8 service nova] [instance: f616a54a-647c-4454-967c-e0ef50552720] Received event network-changed-3cd0eb57-3579-49ff-8787-df8d6a4eadab {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 993.340220] env[64020]: DEBUG nova.compute.manager [req-712e4443-8b07-4967-8ed3-5fc9cfa9e1f6 req-93775ee9-bb32-4f15-a281-edb0795120b8 service nova] [instance: f616a54a-647c-4454-967c-e0ef50552720] Refreshing instance network info cache due to event network-changed-3cd0eb57-3579-49ff-8787-df8d6a4eadab. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 993.340691] env[64020]: DEBUG oslo_concurrency.lockutils [req-712e4443-8b07-4967-8ed3-5fc9cfa9e1f6 req-93775ee9-bb32-4f15-a281-edb0795120b8 service nova] Acquiring lock "refresh_cache-f616a54a-647c-4454-967c-e0ef50552720" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 993.340691] env[64020]: DEBUG oslo_concurrency.lockutils [req-712e4443-8b07-4967-8ed3-5fc9cfa9e1f6 req-93775ee9-bb32-4f15-a281-edb0795120b8 service nova] Acquired lock "refresh_cache-f616a54a-647c-4454-967c-e0ef50552720" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 993.340691] env[64020]: DEBUG nova.network.neutron [req-712e4443-8b07-4967-8ed3-5fc9cfa9e1f6 req-93775ee9-bb32-4f15-a281-edb0795120b8 service nova] [instance: f616a54a-647c-4454-967c-e0ef50552720] Refreshing network info cache for port 3cd0eb57-3579-49ff-8787-df8d6a4eadab {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 993.341036] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-712e4443-8b07-4967-8ed3-5fc9cfa9e1f6 req-93775ee9-bb32-4f15-a281-edb0795120b8 service nova] Expecting reply to msg 83ed297d6f544dd3a93213e3d40584ba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 993.351131] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bdb7c1c2feae47a3a5bfdf79ff0e8c5e [ 993.356319] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 83ed297d6f544dd3a93213e3d40584ba [ 993.361267] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b0468ae0ee764072859eed3bfad7fe93 [ 993.486800] env[64020]: ERROR nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3cd0eb57-3579-49ff-8787-df8d6a4eadab, please check neutron logs for more information. [ 993.486800] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 993.486800] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 993.486800] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 993.486800] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 993.486800] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 993.486800] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 993.486800] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 993.486800] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 993.486800] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 993.486800] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 993.486800] env[64020]: ERROR nova.compute.manager raise self.value [ 993.486800] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 993.486800] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 993.486800] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 993.486800] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 993.487637] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 993.487637] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 993.487637] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3cd0eb57-3579-49ff-8787-df8d6a4eadab, please check neutron logs for more information. [ 993.487637] env[64020]: ERROR nova.compute.manager [ 993.487637] env[64020]: Traceback (most recent call last): [ 993.487637] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 993.487637] env[64020]: listener.cb(fileno) [ 993.487637] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 993.487637] env[64020]: result = function(*args, **kwargs) [ 993.487637] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 993.487637] env[64020]: return func(*args, **kwargs) [ 993.487637] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 993.487637] env[64020]: raise e [ 993.487637] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 993.487637] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 993.487637] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 993.487637] env[64020]: created_port_ids = self._update_ports_for_instance( [ 993.487637] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 993.487637] env[64020]: with excutils.save_and_reraise_exception(): [ 993.487637] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 993.487637] env[64020]: self.force_reraise() [ 993.487637] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 993.487637] env[64020]: raise self.value [ 993.487637] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 993.487637] env[64020]: updated_port = self._update_port( [ 993.487637] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 993.487637] env[64020]: _ensure_no_port_binding_failure(port) [ 993.487637] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 993.487637] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 993.488605] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 3cd0eb57-3579-49ff-8787-df8d6a4eadab, please check neutron logs for more information. [ 993.488605] env[64020]: Removing descriptor: 16 [ 993.632023] env[64020]: DEBUG nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 993.660460] env[64020]: DEBUG nova.virt.hardware [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 993.660704] env[64020]: DEBUG nova.virt.hardware [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 993.660859] env[64020]: DEBUG nova.virt.hardware [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 993.661037] env[64020]: DEBUG nova.virt.hardware [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 993.661178] env[64020]: DEBUG nova.virt.hardware [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 993.661352] env[64020]: DEBUG nova.virt.hardware [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 993.661567] env[64020]: DEBUG nova.virt.hardware [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 993.661723] env[64020]: DEBUG nova.virt.hardware [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 993.661885] env[64020]: DEBUG nova.virt.hardware [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 993.662044] env[64020]: DEBUG nova.virt.hardware [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 993.662245] env[64020]: DEBUG nova.virt.hardware [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 993.663088] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d021a93-7a12-4099-bea4-c94c1f2b27b9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 993.670964] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42290d0f-9b46-441e-a244-3fdd6f9a1705 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 993.686908] env[64020]: ERROR nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3cd0eb57-3579-49ff-8787-df8d6a4eadab, please check neutron logs for more information. [ 993.686908] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] Traceback (most recent call last): [ 993.686908] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 993.686908] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] yield resources [ 993.686908] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 993.686908] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] self.driver.spawn(context, instance, image_meta, [ 993.686908] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 993.686908] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] self._vmops.spawn(context, instance, image_meta, injected_files, [ 993.686908] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 993.686908] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] vm_ref = self.build_virtual_machine(instance, [ 993.686908] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] vif_infos = vmwarevif.get_vif_info(self._session, [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] for vif in network_info: [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] return self._sync_wrapper(fn, *args, **kwargs) [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] self.wait() [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] self[:] = self._gt.wait() [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] return self._exit_event.wait() [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 993.687247] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] current.throw(*self._exc) [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] result = function(*args, **kwargs) [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] return func(*args, **kwargs) [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] raise e [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] nwinfo = self.network_api.allocate_for_instance( [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] created_port_ids = self._update_ports_for_instance( [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] with excutils.save_and_reraise_exception(): [ 993.687605] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 993.687966] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] self.force_reraise() [ 993.687966] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 993.687966] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] raise self.value [ 993.687966] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 993.687966] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] updated_port = self._update_port( [ 993.687966] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 993.687966] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] _ensure_no_port_binding_failure(port) [ 993.687966] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 993.687966] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] raise exception.PortBindingFailed(port_id=port['id']) [ 993.687966] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] nova.exception.PortBindingFailed: Binding failed for port 3cd0eb57-3579-49ff-8787-df8d6a4eadab, please check neutron logs for more information. [ 993.687966] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] [ 993.687966] env[64020]: INFO nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Terminating instance [ 993.689185] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquiring lock "refresh_cache-f616a54a-647c-4454-967c-e0ef50552720" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 993.831330] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 993.843573] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.238s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 993.844133] env[64020]: DEBUG nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 993.846006] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 5fcd0333a4ee416b810035fcd279c267 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 993.846911] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 13.884s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 993.847122] env[64020]: DEBUG nova.objects.instance [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lazy-loading 'resources' on Instance uuid 77801ae9-89db-4dc7-af03-0646af73b121 {{(pid=64020) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 993.847499] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg d0803b6ef73b451ba5f03dfd34981b5b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 993.854348] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0803b6ef73b451ba5f03dfd34981b5b [ 993.883740] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5fcd0333a4ee416b810035fcd279c267 [ 993.884919] env[64020]: DEBUG nova.network.neutron [req-712e4443-8b07-4967-8ed3-5fc9cfa9e1f6 req-93775ee9-bb32-4f15-a281-edb0795120b8 service nova] [instance: f616a54a-647c-4454-967c-e0ef50552720] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 994.032956] env[64020]: DEBUG nova.network.neutron [req-712e4443-8b07-4967-8ed3-5fc9cfa9e1f6 req-93775ee9-bb32-4f15-a281-edb0795120b8 service nova] [instance: f616a54a-647c-4454-967c-e0ef50552720] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 994.033433] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-712e4443-8b07-4967-8ed3-5fc9cfa9e1f6 req-93775ee9-bb32-4f15-a281-edb0795120b8 service nova] Expecting reply to msg 2b61c62a700c41ff9996b009e56d3ffb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 994.043861] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b61c62a700c41ff9996b009e56d3ffb [ 994.354663] env[64020]: DEBUG nova.compute.utils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 994.355262] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 1680d6e7205b4bca97721daa3c3c43d7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 994.356471] env[64020]: DEBUG nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 994.356716] env[64020]: DEBUG nova.network.neutron [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 994.374353] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1680d6e7205b4bca97721daa3c3c43d7 [ 994.455249] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ff7fd45-60c1-48ca-a59c-154d16689b33 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.463395] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33b76bc0-7c1b-4f66-9a5f-9c1a2d2c9599 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.494190] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-467c6a50-8167-4fb7-bd29-5315eb2bb295 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.502510] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83d2befc-fce0-42ff-a66b-c3da5906d46d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.517242] env[64020]: DEBUG nova.compute.provider_tree [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 994.517890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 8aaf90db56b347e1b4009849ea1121f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 994.525474] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8aaf90db56b347e1b4009849ea1121f5 [ 994.536097] env[64020]: DEBUG oslo_concurrency.lockutils [req-712e4443-8b07-4967-8ed3-5fc9cfa9e1f6 req-93775ee9-bb32-4f15-a281-edb0795120b8 service nova] Releasing lock "refresh_cache-f616a54a-647c-4454-967c-e0ef50552720" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 994.536531] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquired lock "refresh_cache-f616a54a-647c-4454-967c-e0ef50552720" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 994.536777] env[64020]: DEBUG nova.network.neutron [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 994.537260] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 391d5b05b57f43c79eb2f0f48ca0f360 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 994.544014] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 391d5b05b57f43c79eb2f0f48ca0f360 [ 994.615799] env[64020]: DEBUG nova.policy [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd496b47ef9db47fa97323baf6bbd63ed', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f1fb1191ee044c4f921f5b7935f0109d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 994.859940] env[64020]: DEBUG nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 994.861647] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg f877991ef20a4b7b93b5097191729831 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 994.885308] env[64020]: DEBUG nova.network.neutron [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Successfully created port: c83da57e-a466-44b2-89db-039f682c7fc1 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 994.914925] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f877991ef20a4b7b93b5097191729831 [ 995.020381] env[64020]: DEBUG nova.scheduler.client.report [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 995.022964] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 9d5654d4938f43a5923604d0b06d9699 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 995.036398] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9d5654d4938f43a5923604d0b06d9699 [ 995.096673] env[64020]: DEBUG nova.network.neutron [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 995.238610] env[64020]: DEBUG nova.network.neutron [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 995.239252] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 4ecacdcdf89b47d78b547fdb288a1fd2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 995.247570] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4ecacdcdf89b47d78b547fdb288a1fd2 [ 995.362299] env[64020]: DEBUG nova.compute.manager [req-bd07fc6c-be8e-43d8-8a17-acf443e8ba1f req-9d80db0e-7140-4117-966e-11a6416b2445 service nova] [instance: f616a54a-647c-4454-967c-e0ef50552720] Received event network-vif-deleted-3cd0eb57-3579-49ff-8787-df8d6a4eadab {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 995.366435] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 2a7887fd54184d3caea5468292fcb7c9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 995.413839] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2a7887fd54184d3caea5468292fcb7c9 [ 995.525812] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.679s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 995.528459] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.755s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 995.530005] env[64020]: INFO nova.compute.claims [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 995.531562] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 2bac6e5af8834c8f892bf4ca823bd414 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 995.550500] env[64020]: INFO nova.scheduler.client.report [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Deleted allocations for instance 77801ae9-89db-4dc7-af03-0646af73b121 [ 995.554045] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg c30cb275cc1f4f3f9520aa5b8abc891f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 995.567638] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2bac6e5af8834c8f892bf4ca823bd414 [ 995.595627] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c30cb275cc1f4f3f9520aa5b8abc891f [ 995.742158] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Releasing lock "refresh_cache-f616a54a-647c-4454-967c-e0ef50552720" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 995.742490] env[64020]: DEBUG nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 995.742693] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 995.742976] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5f3aad81-de96-4223-8dd0-d1aff2cf3ece {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 995.752321] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4581025-2364-4bb7-8edf-22e49fe71463 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 995.774674] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f616a54a-647c-4454-967c-e0ef50552720 could not be found. [ 995.774904] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 995.775083] env[64020]: INFO nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Took 0.03 seconds to destroy the instance on the hypervisor. [ 995.775335] env[64020]: DEBUG oslo.service.loopingcall [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 995.775568] env[64020]: DEBUG nova.compute.manager [-] [instance: f616a54a-647c-4454-967c-e0ef50552720] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 995.775654] env[64020]: DEBUG nova.network.neutron [-] [instance: f616a54a-647c-4454-967c-e0ef50552720] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 995.816312] env[64020]: DEBUG nova.network.neutron [-] [instance: f616a54a-647c-4454-967c-e0ef50552720] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 995.816848] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3e35f2a7a14b4fee81f3044bfa5d6643 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 995.825978] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e35f2a7a14b4fee81f3044bfa5d6643 [ 995.869752] env[64020]: DEBUG nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 995.897840] env[64020]: DEBUG nova.virt.hardware [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 995.898177] env[64020]: DEBUG nova.virt.hardware [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 995.898395] env[64020]: DEBUG nova.virt.hardware [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 995.898682] env[64020]: DEBUG nova.virt.hardware [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 995.898887] env[64020]: DEBUG nova.virt.hardware [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 995.899088] env[64020]: DEBUG nova.virt.hardware [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 995.899411] env[64020]: DEBUG nova.virt.hardware [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 995.899639] env[64020]: DEBUG nova.virt.hardware [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 995.899899] env[64020]: DEBUG nova.virt.hardware [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 995.900138] env[64020]: DEBUG nova.virt.hardware [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 995.900375] env[64020]: DEBUG nova.virt.hardware [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 995.901317] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94eb0949-8e25-441d-8d9f-c0bc61f00d61 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 995.909483] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-065653f5-2fd7-4c83-b377-ed815bd09211 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.027567] env[64020]: ERROR nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c83da57e-a466-44b2-89db-039f682c7fc1, please check neutron logs for more information. [ 996.027567] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 996.027567] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 996.027567] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 996.027567] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 996.027567] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 996.027567] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 996.027567] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 996.027567] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 996.027567] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 996.027567] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 996.027567] env[64020]: ERROR nova.compute.manager raise self.value [ 996.027567] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 996.027567] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 996.027567] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 996.027567] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 996.028164] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 996.028164] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 996.028164] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c83da57e-a466-44b2-89db-039f682c7fc1, please check neutron logs for more information. [ 996.028164] env[64020]: ERROR nova.compute.manager [ 996.028164] env[64020]: Traceback (most recent call last): [ 996.028164] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 996.028164] env[64020]: listener.cb(fileno) [ 996.028164] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 996.028164] env[64020]: result = function(*args, **kwargs) [ 996.028164] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 996.028164] env[64020]: return func(*args, **kwargs) [ 996.028164] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 996.028164] env[64020]: raise e [ 996.028164] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 996.028164] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 996.028164] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 996.028164] env[64020]: created_port_ids = self._update_ports_for_instance( [ 996.028164] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 996.028164] env[64020]: with excutils.save_and_reraise_exception(): [ 996.028164] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 996.028164] env[64020]: self.force_reraise() [ 996.028164] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 996.028164] env[64020]: raise self.value [ 996.028164] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 996.028164] env[64020]: updated_port = self._update_port( [ 996.028164] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 996.028164] env[64020]: _ensure_no_port_binding_failure(port) [ 996.028164] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 996.028164] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 996.029091] env[64020]: nova.exception.PortBindingFailed: Binding failed for port c83da57e-a466-44b2-89db-039f682c7fc1, please check neutron logs for more information. [ 996.029091] env[64020]: Removing descriptor: 16 [ 996.029359] env[64020]: ERROR nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c83da57e-a466-44b2-89db-039f682c7fc1, please check neutron logs for more information. [ 996.029359] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Traceback (most recent call last): [ 996.029359] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 996.029359] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] yield resources [ 996.029359] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 996.029359] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] self.driver.spawn(context, instance, image_meta, [ 996.029359] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 996.029359] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 996.029359] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 996.029359] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] vm_ref = self.build_virtual_machine(instance, [ 996.029359] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] vif_infos = vmwarevif.get_vif_info(self._session, [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] for vif in network_info: [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] return self._sync_wrapper(fn, *args, **kwargs) [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] self.wait() [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] self[:] = self._gt.wait() [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] return self._exit_event.wait() [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 996.029763] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] result = hub.switch() [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] return self.greenlet.switch() [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] result = function(*args, **kwargs) [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] return func(*args, **kwargs) [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] raise e [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] nwinfo = self.network_api.allocate_for_instance( [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] created_port_ids = self._update_ports_for_instance( [ 996.030162] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] with excutils.save_and_reraise_exception(): [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] self.force_reraise() [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] raise self.value [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] updated_port = self._update_port( [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] _ensure_no_port_binding_failure(port) [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] raise exception.PortBindingFailed(port_id=port['id']) [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] nova.exception.PortBindingFailed: Binding failed for port c83da57e-a466-44b2-89db-039f682c7fc1, please check neutron logs for more information. [ 996.030609] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] [ 996.031005] env[64020]: INFO nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Terminating instance [ 996.033129] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-bd872d96-d664-4ecb-ad59-516d95ad0cb7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 996.033318] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-bd872d96-d664-4ecb-ad59-516d95ad0cb7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 996.033648] env[64020]: DEBUG nova.network.neutron [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 996.034216] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 8f907086217245dd9ceea2d491acb459 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 996.041695] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f907086217245dd9ceea2d491acb459 [ 996.045507] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg bc2879f6bd9c419daa4e0de27a6172c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 996.053638] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc2879f6bd9c419daa4e0de27a6172c1 [ 996.057608] env[64020]: DEBUG oslo_concurrency.lockutils [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Lock "77801ae9-89db-4dc7-af03-0646af73b121" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 19.851s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 996.058054] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-d496e1e3-4034-4520-b13a-427f561cc4a0 tempest-ServerShowV247Test-984902028 tempest-ServerShowV247Test-984902028-project-member] Expecting reply to msg 248d1dbb0baa47ffb229edadfa8b7e5e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 996.073044] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 248d1dbb0baa47ffb229edadfa8b7e5e [ 996.319331] env[64020]: DEBUG nova.network.neutron [-] [instance: f616a54a-647c-4454-967c-e0ef50552720] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 996.319911] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg afb01cc36f19417ab4d0adac34e90461 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 996.327747] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg afb01cc36f19417ab4d0adac34e90461 [ 996.560782] env[64020]: DEBUG nova.network.neutron [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 996.640586] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c749d367-2eab-4b9b-b4c5-624191a499df {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.648890] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9005c7e4-20aa-49ca-8aa2-4542fddc5fec {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.652442] env[64020]: DEBUG nova.network.neutron [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 996.653039] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 9825746876a049068feeda8d78d6edad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 996.680404] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9825746876a049068feeda8d78d6edad [ 996.681948] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-813a7dfe-84ff-4cff-96ec-5681a185f4ff {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.689376] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fb871f8-fe34-4491-9448-27ac05a2bf3b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.703389] env[64020]: DEBUG nova.compute.provider_tree [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 996.703920] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 91dbe91dfe924ed79efbd8c7b5227320 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 996.711262] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91dbe91dfe924ed79efbd8c7b5227320 [ 996.821965] env[64020]: INFO nova.compute.manager [-] [instance: f616a54a-647c-4454-967c-e0ef50552720] Took 1.05 seconds to deallocate network for instance. [ 996.824667] env[64020]: DEBUG nova.compute.claims [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 996.824949] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 997.155734] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-bd872d96-d664-4ecb-ad59-516d95ad0cb7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 997.156201] env[64020]: DEBUG nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 997.156402] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 997.156712] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5ed5be9c-4226-4faa-adf1-9ab67de34523 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 997.167038] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0802642f-8cb2-47c8-b383-1771992a4181 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 997.191482] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance bd872d96-d664-4ecb-ad59-516d95ad0cb7 could not be found. [ 997.191482] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 997.191482] env[64020]: INFO nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Took 0.03 seconds to destroy the instance on the hypervisor. [ 997.191482] env[64020]: DEBUG oslo.service.loopingcall [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 997.191482] env[64020]: DEBUG nova.compute.manager [-] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 997.191482] env[64020]: DEBUG nova.network.neutron [-] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 997.206337] env[64020]: DEBUG nova.network.neutron [-] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 997.206850] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg d37ea71dba5e4afd874ecfed4ef4e11b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 997.208713] env[64020]: DEBUG nova.scheduler.client.report [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 997.211455] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 8cd9be2a4db448e29cb1f705f3d51345 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 997.213726] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d37ea71dba5e4afd874ecfed4ef4e11b [ 997.223953] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8cd9be2a4db448e29cb1f705f3d51345 [ 997.393100] env[64020]: DEBUG nova.compute.manager [req-4f764737-0aa9-4cd9-a150-127205fa274d req-0f897f3b-d0ad-4358-9f69-eea91eef688f service nova] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Received event network-changed-c83da57e-a466-44b2-89db-039f682c7fc1 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 997.393283] env[64020]: DEBUG nova.compute.manager [req-4f764737-0aa9-4cd9-a150-127205fa274d req-0f897f3b-d0ad-4358-9f69-eea91eef688f service nova] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Refreshing instance network info cache due to event network-changed-c83da57e-a466-44b2-89db-039f682c7fc1. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 997.393482] env[64020]: DEBUG oslo_concurrency.lockutils [req-4f764737-0aa9-4cd9-a150-127205fa274d req-0f897f3b-d0ad-4358-9f69-eea91eef688f service nova] Acquiring lock "refresh_cache-bd872d96-d664-4ecb-ad59-516d95ad0cb7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 997.393622] env[64020]: DEBUG oslo_concurrency.lockutils [req-4f764737-0aa9-4cd9-a150-127205fa274d req-0f897f3b-d0ad-4358-9f69-eea91eef688f service nova] Acquired lock "refresh_cache-bd872d96-d664-4ecb-ad59-516d95ad0cb7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 997.393775] env[64020]: DEBUG nova.network.neutron [req-4f764737-0aa9-4cd9-a150-127205fa274d req-0f897f3b-d0ad-4358-9f69-eea91eef688f service nova] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Refreshing network info cache for port c83da57e-a466-44b2-89db-039f682c7fc1 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 997.394237] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-4f764737-0aa9-4cd9-a150-127205fa274d req-0f897f3b-d0ad-4358-9f69-eea91eef688f service nova] Expecting reply to msg 436ece6ced064973a5bb807a259c0d87 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 997.403503] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 436ece6ced064973a5bb807a259c0d87 [ 997.713103] env[64020]: DEBUG nova.network.neutron [-] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 997.713568] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 89fee09093c445528e88d5749168d141 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 997.714818] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.186s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 997.715279] env[64020]: DEBUG nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 997.716897] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg e645ff97252440329c4fe55517efb9d2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 997.718072] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.967s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 997.719705] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg b0dc3ca9fc494d24ad539ad465bf3ecc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 997.733749] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89fee09093c445528e88d5749168d141 [ 997.758420] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e645ff97252440329c4fe55517efb9d2 [ 997.758956] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b0dc3ca9fc494d24ad539ad465bf3ecc [ 997.911630] env[64020]: DEBUG nova.network.neutron [req-4f764737-0aa9-4cd9-a150-127205fa274d req-0f897f3b-d0ad-4358-9f69-eea91eef688f service nova] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 998.087467] env[64020]: DEBUG nova.network.neutron [req-4f764737-0aa9-4cd9-a150-127205fa274d req-0f897f3b-d0ad-4358-9f69-eea91eef688f service nova] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 998.088061] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-4f764737-0aa9-4cd9-a150-127205fa274d req-0f897f3b-d0ad-4358-9f69-eea91eef688f service nova] Expecting reply to msg 9342fe272aa941daaa3229dbb5e3e330 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 998.097089] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9342fe272aa941daaa3229dbb5e3e330 [ 998.218726] env[64020]: INFO nova.compute.manager [-] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Took 1.03 seconds to deallocate network for instance. [ 998.221038] env[64020]: DEBUG nova.compute.claims [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 998.221227] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.222334] env[64020]: DEBUG nova.compute.utils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 998.222905] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg e3cd4654039241df9591606e857cbfc0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 998.226473] env[64020]: DEBUG nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 998.226641] env[64020]: DEBUG nova.network.neutron [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 998.235979] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3cd4654039241df9591606e857cbfc0 [ 998.309968] env[64020]: DEBUG nova.policy [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02b993f5bbd5442a8d2c06cff095e867', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e0c4cb7c0e9941f7a056fef708be822c', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 998.334628] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-859b2753-33b3-489e-90ac-2d72a7241007 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.343126] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbf4e644-6228-4382-af8a-0a4eb346a022 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.376581] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0cbc6da2-4604-4a5f-839c-e4614c0c4b71 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.384302] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec34669e-5f53-48e1-9986-69f377041cfc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.398859] env[64020]: DEBUG nova.compute.provider_tree [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 998.399583] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 9b2f133372e242debb621f6189f37d24 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 998.407185] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b2f133372e242debb621f6189f37d24 [ 998.590394] env[64020]: DEBUG oslo_concurrency.lockutils [req-4f764737-0aa9-4cd9-a150-127205fa274d req-0f897f3b-d0ad-4358-9f69-eea91eef688f service nova] Releasing lock "refresh_cache-bd872d96-d664-4ecb-ad59-516d95ad0cb7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 998.590905] env[64020]: DEBUG nova.compute.manager [req-4f764737-0aa9-4cd9-a150-127205fa274d req-0f897f3b-d0ad-4358-9f69-eea91eef688f service nova] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Received event network-vif-deleted-c83da57e-a466-44b2-89db-039f682c7fc1 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 998.609100] env[64020]: DEBUG nova.network.neutron [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Successfully created port: 63fa9007-0069-463c-a79b-3bd3c6dcb3ce {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 998.727659] env[64020]: DEBUG nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 998.729554] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 6e152551402f4a82854622ea987f0c94 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 998.775930] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6e152551402f4a82854622ea987f0c94 [ 998.902435] env[64020]: DEBUG nova.scheduler.client.report [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 998.904932] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 3fee6f54c2244146a16b3fb0d9927f62 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 998.917166] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3fee6f54c2244146a16b3fb0d9927f62 [ 999.234097] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 7addb4d15b2345ec97a645ff38b7546c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 999.269188] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7addb4d15b2345ec97a645ff38b7546c [ 999.407758] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.690s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 999.408471] env[64020]: ERROR nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b23be879-2ee0-4d32-ba16-5aa71dcf946f, please check neutron logs for more information. [ 999.408471] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Traceback (most recent call last): [ 999.408471] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 999.408471] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] self.driver.spawn(context, instance, image_meta, [ 999.408471] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 999.408471] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 999.408471] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 999.408471] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] vm_ref = self.build_virtual_machine(instance, [ 999.408471] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 999.408471] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] vif_infos = vmwarevif.get_vif_info(self._session, [ 999.408471] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] for vif in network_info: [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] return self._sync_wrapper(fn, *args, **kwargs) [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] self.wait() [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] self[:] = self._gt.wait() [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] return self._exit_event.wait() [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] result = hub.switch() [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 999.408826] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] return self.greenlet.switch() [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] result = function(*args, **kwargs) [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] return func(*args, **kwargs) [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] raise e [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] nwinfo = self.network_api.allocate_for_instance( [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] created_port_ids = self._update_ports_for_instance( [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] with excutils.save_and_reraise_exception(): [ 999.409408] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 999.409823] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] self.force_reraise() [ 999.409823] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 999.409823] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] raise self.value [ 999.409823] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 999.409823] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] updated_port = self._update_port( [ 999.409823] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 999.409823] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] _ensure_no_port_binding_failure(port) [ 999.409823] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 999.409823] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] raise exception.PortBindingFailed(port_id=port['id']) [ 999.409823] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] nova.exception.PortBindingFailed: Binding failed for port b23be879-2ee0-4d32-ba16-5aa71dcf946f, please check neutron logs for more information. [ 999.409823] env[64020]: ERROR nova.compute.manager [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] [ 999.410155] env[64020]: DEBUG nova.compute.utils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Binding failed for port b23be879-2ee0-4d32-ba16-5aa71dcf946f, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 999.410423] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.674s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 999.412063] env[64020]: INFO nova.compute.claims [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 999.413879] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 99d8d044ab5b4d95bdf0802bb3669005 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 999.414946] env[64020]: DEBUG nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Build of instance 3c6ddf08-d0f3-444b-8249-f9dabeeef87f was re-scheduled: Binding failed for port b23be879-2ee0-4d32-ba16-5aa71dcf946f, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 999.415393] env[64020]: DEBUG nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 999.415610] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "refresh_cache-3c6ddf08-d0f3-444b-8249-f9dabeeef87f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 999.415752] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquired lock "refresh_cache-3c6ddf08-d0f3-444b-8249-f9dabeeef87f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 999.415968] env[64020]: DEBUG nova.network.neutron [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 999.416357] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 9c6ef9cb8e044c7d9d3f9bee1c8e419d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 999.421250] env[64020]: ERROR nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 63fa9007-0069-463c-a79b-3bd3c6dcb3ce, please check neutron logs for more information. [ 999.421250] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 999.421250] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 999.421250] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 999.421250] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 999.421250] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 999.421250] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 999.421250] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 999.421250] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 999.421250] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 999.421250] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 999.421250] env[64020]: ERROR nova.compute.manager raise self.value [ 999.421250] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 999.421250] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 999.421250] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 999.421250] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 999.421765] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 999.421765] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 999.421765] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 63fa9007-0069-463c-a79b-3bd3c6dcb3ce, please check neutron logs for more information. [ 999.421765] env[64020]: ERROR nova.compute.manager [ 999.421765] env[64020]: Traceback (most recent call last): [ 999.421765] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 999.421765] env[64020]: listener.cb(fileno) [ 999.421765] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 999.421765] env[64020]: result = function(*args, **kwargs) [ 999.421765] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 999.421765] env[64020]: return func(*args, **kwargs) [ 999.421765] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 999.421765] env[64020]: raise e [ 999.421765] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 999.421765] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 999.421765] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 999.421765] env[64020]: created_port_ids = self._update_ports_for_instance( [ 999.421765] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 999.421765] env[64020]: with excutils.save_and_reraise_exception(): [ 999.421765] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 999.421765] env[64020]: self.force_reraise() [ 999.421765] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 999.421765] env[64020]: raise self.value [ 999.421765] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 999.421765] env[64020]: updated_port = self._update_port( [ 999.421765] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 999.421765] env[64020]: _ensure_no_port_binding_failure(port) [ 999.421765] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 999.421765] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 999.422699] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 63fa9007-0069-463c-a79b-3bd3c6dcb3ce, please check neutron logs for more information. [ 999.422699] env[64020]: Removing descriptor: 16 [ 999.422699] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c6ef9cb8e044c7d9d3f9bee1c8e419d [ 999.425765] env[64020]: DEBUG nova.compute.manager [req-b56262ab-12f6-4971-b20f-849b0a6badaa req-d3b4076f-fc63-428a-b8ae-326ccb135076 service nova] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Received event network-changed-63fa9007-0069-463c-a79b-3bd3c6dcb3ce {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 999.425942] env[64020]: DEBUG nova.compute.manager [req-b56262ab-12f6-4971-b20f-849b0a6badaa req-d3b4076f-fc63-428a-b8ae-326ccb135076 service nova] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Refreshing instance network info cache due to event network-changed-63fa9007-0069-463c-a79b-3bd3c6dcb3ce. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 999.426143] env[64020]: DEBUG oslo_concurrency.lockutils [req-b56262ab-12f6-4971-b20f-849b0a6badaa req-d3b4076f-fc63-428a-b8ae-326ccb135076 service nova] Acquiring lock "refresh_cache-6db545fe-881e-4c9a-b514-bd98d7528674" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 999.426279] env[64020]: DEBUG oslo_concurrency.lockutils [req-b56262ab-12f6-4971-b20f-849b0a6badaa req-d3b4076f-fc63-428a-b8ae-326ccb135076 service nova] Acquired lock "refresh_cache-6db545fe-881e-4c9a-b514-bd98d7528674" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 999.426458] env[64020]: DEBUG nova.network.neutron [req-b56262ab-12f6-4971-b20f-849b0a6badaa req-d3b4076f-fc63-428a-b8ae-326ccb135076 service nova] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Refreshing network info cache for port 63fa9007-0069-463c-a79b-3bd3c6dcb3ce {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 999.426847] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-b56262ab-12f6-4971-b20f-849b0a6badaa req-d3b4076f-fc63-428a-b8ae-326ccb135076 service nova] Expecting reply to msg 4bf88f068364499cb6d651b6a980f534 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 999.432259] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4bf88f068364499cb6d651b6a980f534 [ 999.457225] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99d8d044ab5b4d95bdf0802bb3669005 [ 999.737136] env[64020]: DEBUG nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 999.764814] env[64020]: DEBUG nova.virt.hardware [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 999.765073] env[64020]: DEBUG nova.virt.hardware [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 999.765261] env[64020]: DEBUG nova.virt.hardware [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 999.765488] env[64020]: DEBUG nova.virt.hardware [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 999.765734] env[64020]: DEBUG nova.virt.hardware [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 999.765903] env[64020]: DEBUG nova.virt.hardware [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 999.766107] env[64020]: DEBUG nova.virt.hardware [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 999.766262] env[64020]: DEBUG nova.virt.hardware [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 999.766430] env[64020]: DEBUG nova.virt.hardware [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 999.766614] env[64020]: DEBUG nova.virt.hardware [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 999.766788] env[64020]: DEBUG nova.virt.hardware [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 999.767619] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0542e300-0bb5-404f-bb60-0351a57dc211 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.775351] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3b5fc80-fca6-44a6-b525-d2bc43ee2e1c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.788340] env[64020]: ERROR nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 63fa9007-0069-463c-a79b-3bd3c6dcb3ce, please check neutron logs for more information. [ 999.788340] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Traceback (most recent call last): [ 999.788340] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 999.788340] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] yield resources [ 999.788340] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 999.788340] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] self.driver.spawn(context, instance, image_meta, [ 999.788340] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 999.788340] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] self._vmops.spawn(context, instance, image_meta, injected_files, [ 999.788340] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 999.788340] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] vm_ref = self.build_virtual_machine(instance, [ 999.788340] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] vif_infos = vmwarevif.get_vif_info(self._session, [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] for vif in network_info: [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] return self._sync_wrapper(fn, *args, **kwargs) [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] self.wait() [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] self[:] = self._gt.wait() [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] return self._exit_event.wait() [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 999.788765] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] current.throw(*self._exc) [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] result = function(*args, **kwargs) [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] return func(*args, **kwargs) [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] raise e [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] nwinfo = self.network_api.allocate_for_instance( [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] created_port_ids = self._update_ports_for_instance( [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] with excutils.save_and_reraise_exception(): [ 999.789188] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 999.789618] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] self.force_reraise() [ 999.789618] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 999.789618] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] raise self.value [ 999.789618] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 999.789618] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] updated_port = self._update_port( [ 999.789618] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 999.789618] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] _ensure_no_port_binding_failure(port) [ 999.789618] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 999.789618] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] raise exception.PortBindingFailed(port_id=port['id']) [ 999.789618] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] nova.exception.PortBindingFailed: Binding failed for port 63fa9007-0069-463c-a79b-3bd3c6dcb3ce, please check neutron logs for more information. [ 999.789618] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] [ 999.789618] env[64020]: INFO nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Terminating instance [ 999.790525] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-6db545fe-881e-4c9a-b514-bd98d7528674" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 999.919738] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg ec56730bf5fc4f5ba86c157a91ac7087 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 999.926677] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec56730bf5fc4f5ba86c157a91ac7087 [ 999.935852] env[64020]: DEBUG nova.network.neutron [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 999.949033] env[64020]: DEBUG nova.network.neutron [req-b56262ab-12f6-4971-b20f-849b0a6badaa req-d3b4076f-fc63-428a-b8ae-326ccb135076 service nova] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1000.074701] env[64020]: DEBUG nova.network.neutron [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1000.075360] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 3d57ba6778fc4ba087e274343d9aa3c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1000.077958] env[64020]: DEBUG nova.network.neutron [req-b56262ab-12f6-4971-b20f-849b0a6badaa req-d3b4076f-fc63-428a-b8ae-326ccb135076 service nova] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1000.078462] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-b56262ab-12f6-4971-b20f-849b0a6badaa req-d3b4076f-fc63-428a-b8ae-326ccb135076 service nova] Expecting reply to msg 1799a0d1bc584495993f3ed86ecf3eb6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1000.084657] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d57ba6778fc4ba087e274343d9aa3c1 [ 1000.086341] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1799a0d1bc584495993f3ed86ecf3eb6 [ 1000.498912] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b87f70a-3b1f-4119-b2f3-84aeb40cac7c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.505921] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-82b1e568-3191-4906-88eb-c162b22fdbeb {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.534377] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e195a70-46b6-4626-b393-c1119a38a861 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.540855] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12f386b1-0099-4b21-8d18-c8d283caf142 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.553113] env[64020]: DEBUG nova.compute.provider_tree [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1000.553580] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 15bb4ccfebe941458378c4f57651f00e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1000.560295] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 15bb4ccfebe941458378c4f57651f00e [ 1000.578197] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Releasing lock "refresh_cache-3c6ddf08-d0f3-444b-8249-f9dabeeef87f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1000.578405] env[64020]: DEBUG nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1000.578582] env[64020]: DEBUG nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1000.578741] env[64020]: DEBUG nova.network.neutron [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1000.580669] env[64020]: DEBUG oslo_concurrency.lockutils [req-b56262ab-12f6-4971-b20f-849b0a6badaa req-d3b4076f-fc63-428a-b8ae-326ccb135076 service nova] Releasing lock "refresh_cache-6db545fe-881e-4c9a-b514-bd98d7528674" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1000.581028] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-6db545fe-881e-4c9a-b514-bd98d7528674" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1000.581196] env[64020]: DEBUG nova.network.neutron [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1000.581717] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg fec9b1e446fc47ac8fa8eaac9ae032ab in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1000.587961] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fec9b1e446fc47ac8fa8eaac9ae032ab [ 1000.598972] env[64020]: DEBUG nova.network.neutron [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1000.599529] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 132f1e9521184daa9731884b1a32a507 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1000.605474] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 132f1e9521184daa9731884b1a32a507 [ 1001.056068] env[64020]: DEBUG nova.scheduler.client.report [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1001.058540] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg c5c0c9e7dc3f4bcaaf1a0dad2d57f3f7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1001.071002] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5c0c9e7dc3f4bcaaf1a0dad2d57f3f7 [ 1001.100663] env[64020]: DEBUG nova.network.neutron [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1001.102959] env[64020]: DEBUG nova.network.neutron [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1001.103603] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 4d4b81d997994de384e8c9505fee186d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1001.111833] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d4b81d997994de384e8c9505fee186d [ 1001.196940] env[64020]: DEBUG nova.network.neutron [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1001.197479] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg cabd65395df64742b26ade49b628c4b8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1001.208498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cabd65395df64742b26ade49b628c4b8 [ 1001.453828] env[64020]: DEBUG nova.compute.manager [req-346c206a-9ff5-4347-a642-3980a325a556 req-be3d6803-bed2-4cc6-865d-6c97aac3b923 service nova] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Received event network-vif-deleted-63fa9007-0069-463c-a79b-3bd3c6dcb3ce {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1001.561905] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.151s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1001.562478] env[64020]: DEBUG nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1001.564363] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 5425344f8eed4b14a4a21b9bac274752 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1001.565974] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 7.735s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1001.567598] env[64020]: INFO nova.compute.claims [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1001.569184] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 99e36e3f040d4055afa7515744603845 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1001.605926] env[64020]: INFO nova.compute.manager [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: 3c6ddf08-d0f3-444b-8249-f9dabeeef87f] Took 1.03 seconds to deallocate network for instance. [ 1001.608049] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 70b3cd88857647c18037b431c94d7865 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1001.609716] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5425344f8eed4b14a4a21b9bac274752 [ 1001.610223] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99e36e3f040d4055afa7515744603845 [ 1001.643068] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 70b3cd88857647c18037b431c94d7865 [ 1001.700060] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-6db545fe-881e-4c9a-b514-bd98d7528674" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1001.700519] env[64020]: DEBUG nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1001.700711] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1001.701029] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2ebc795b-ee0a-40b9-a3fc-d807093c6d3d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1001.709886] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-561780ee-10d7-4b57-9005-a8de3d6c8ad1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1001.732513] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6db545fe-881e-4c9a-b514-bd98d7528674 could not be found. [ 1001.732830] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1001.733093] env[64020]: INFO nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1001.733444] env[64020]: DEBUG oslo.service.loopingcall [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1001.733716] env[64020]: DEBUG nova.compute.manager [-] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1001.733844] env[64020]: DEBUG nova.network.neutron [-] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1001.751575] env[64020]: DEBUG nova.network.neutron [-] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1001.752266] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c8cbc2fd96c14dec9f168f8c63e75856 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1001.759262] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c8cbc2fd96c14dec9f168f8c63e75856 [ 1002.068104] env[64020]: DEBUG nova.compute.utils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1002.068844] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 0772be70a4d247a1b41da629bc1f20d9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1002.069921] env[64020]: DEBUG nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1002.070132] env[64020]: DEBUG nova.network.neutron [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1002.076108] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 5d607f30ff884805a856954690a992d0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1002.083745] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0772be70a4d247a1b41da629bc1f20d9 [ 1002.084803] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d607f30ff884805a856954690a992d0 [ 1002.114802] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 4f4ce45e336340bca33d0d3e2f1e23ca in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1002.155198] env[64020]: DEBUG nova.policy [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '0db3b318498e43f487968e56d08d64ac', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '89227a81e91c40928fc2d6c398c8eeca', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1002.168280] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f4ce45e336340bca33d0d3e2f1e23ca [ 1002.254475] env[64020]: DEBUG nova.network.neutron [-] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1002.254900] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 752e2139d9f54929975ae5168f653aff in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1002.264728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 752e2139d9f54929975ae5168f653aff [ 1002.516166] env[64020]: DEBUG nova.network.neutron [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Successfully created port: 23ff4de9-4b22-4c53-afb4-ebdf8d8167b4 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1002.577216] env[64020]: DEBUG nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1002.577216] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 422a397456954836b0c5a665fa583936 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1002.615187] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 422a397456954836b0c5a665fa583936 [ 1002.635955] env[64020]: INFO nova.scheduler.client.report [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Deleted allocations for instance 3c6ddf08-d0f3-444b-8249-f9dabeeef87f [ 1002.641951] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 2d5622ee8eda4815a267b52047b8b7b1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1002.655797] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d5622ee8eda4815a267b52047b8b7b1 [ 1002.697478] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46eb0b42-5fc1-4001-944a-2237fa39a95a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1002.705940] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e5a2a201-16e6-45af-bbe2-e57ed661bd32 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1002.744865] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f97f073-58cf-4877-8bc5-3d9e529e3e59 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1002.752782] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ff5bd2e-1ca0-490b-b6db-cedeb578e657 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1002.757024] env[64020]: INFO nova.compute.manager [-] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Took 1.02 seconds to deallocate network for instance. [ 1002.759223] env[64020]: DEBUG nova.compute.claims [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1002.759396] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1002.767168] env[64020]: DEBUG nova.compute.provider_tree [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1002.767599] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 88f0a0349e0f4d32aeac895b75ca6e13 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1002.775485] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88f0a0349e0f4d32aeac895b75ca6e13 [ 1003.081904] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg c014cbfff67a441183eba72d82c08118 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1003.123706] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c014cbfff67a441183eba72d82c08118 [ 1003.144249] env[64020]: DEBUG oslo_concurrency.lockutils [None req-0a7eeffb-a5f8-4ec8-9aff-924d5dc67616 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "3c6ddf08-d0f3-444b-8249-f9dabeeef87f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 74.358s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1003.270058] env[64020]: DEBUG nova.scheduler.client.report [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1003.272574] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg cc225b848ac04bdb9499d260662a8157 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1003.283516] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cc225b848ac04bdb9499d260662a8157 [ 1003.403849] env[64020]: ERROR nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 23ff4de9-4b22-4c53-afb4-ebdf8d8167b4, please check neutron logs for more information. [ 1003.403849] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1003.403849] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1003.403849] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1003.403849] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1003.403849] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1003.403849] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1003.403849] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1003.403849] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1003.403849] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1003.403849] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1003.403849] env[64020]: ERROR nova.compute.manager raise self.value [ 1003.403849] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1003.403849] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1003.403849] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1003.403849] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1003.404395] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1003.404395] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1003.404395] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 23ff4de9-4b22-4c53-afb4-ebdf8d8167b4, please check neutron logs for more information. [ 1003.404395] env[64020]: ERROR nova.compute.manager [ 1003.404395] env[64020]: Traceback (most recent call last): [ 1003.404395] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1003.404395] env[64020]: listener.cb(fileno) [ 1003.404395] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1003.404395] env[64020]: result = function(*args, **kwargs) [ 1003.404395] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1003.404395] env[64020]: return func(*args, **kwargs) [ 1003.404395] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1003.404395] env[64020]: raise e [ 1003.404395] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1003.404395] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1003.404395] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1003.404395] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1003.404395] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1003.404395] env[64020]: with excutils.save_and_reraise_exception(): [ 1003.404395] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1003.404395] env[64020]: self.force_reraise() [ 1003.404395] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1003.404395] env[64020]: raise self.value [ 1003.404395] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1003.404395] env[64020]: updated_port = self._update_port( [ 1003.404395] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1003.404395] env[64020]: _ensure_no_port_binding_failure(port) [ 1003.404395] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1003.404395] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1003.405360] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 23ff4de9-4b22-4c53-afb4-ebdf8d8167b4, please check neutron logs for more information. [ 1003.405360] env[64020]: Removing descriptor: 17 [ 1003.476890] env[64020]: DEBUG nova.compute.manager [req-5557a85d-4a19-4b5c-ae76-559768a7b2e7 req-70ca18c0-26dc-4ac8-8b1b-00846d592f6e service nova] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Received event network-changed-23ff4de9-4b22-4c53-afb4-ebdf8d8167b4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1003.477079] env[64020]: DEBUG nova.compute.manager [req-5557a85d-4a19-4b5c-ae76-559768a7b2e7 req-70ca18c0-26dc-4ac8-8b1b-00846d592f6e service nova] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Refreshing instance network info cache due to event network-changed-23ff4de9-4b22-4c53-afb4-ebdf8d8167b4. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1003.477301] env[64020]: DEBUG oslo_concurrency.lockutils [req-5557a85d-4a19-4b5c-ae76-559768a7b2e7 req-70ca18c0-26dc-4ac8-8b1b-00846d592f6e service nova] Acquiring lock "refresh_cache-8f90d1e0-6ede-4e89-8423-170c0d0e605f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1003.477443] env[64020]: DEBUG oslo_concurrency.lockutils [req-5557a85d-4a19-4b5c-ae76-559768a7b2e7 req-70ca18c0-26dc-4ac8-8b1b-00846d592f6e service nova] Acquired lock "refresh_cache-8f90d1e0-6ede-4e89-8423-170c0d0e605f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1003.477608] env[64020]: DEBUG nova.network.neutron [req-5557a85d-4a19-4b5c-ae76-559768a7b2e7 req-70ca18c0-26dc-4ac8-8b1b-00846d592f6e service nova] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Refreshing network info cache for port 23ff4de9-4b22-4c53-afb4-ebdf8d8167b4 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1003.478031] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-5557a85d-4a19-4b5c-ae76-559768a7b2e7 req-70ca18c0-26dc-4ac8-8b1b-00846d592f6e service nova] Expecting reply to msg c3fb8df1357f4c64b9f086967883ed7a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1003.484594] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3fb8df1357f4c64b9f086967883ed7a [ 1003.585153] env[64020]: DEBUG nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1003.609792] env[64020]: DEBUG nova.virt.hardware [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1003.610021] env[64020]: DEBUG nova.virt.hardware [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1003.610171] env[64020]: DEBUG nova.virt.hardware [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1003.610353] env[64020]: DEBUG nova.virt.hardware [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1003.610492] env[64020]: DEBUG nova.virt.hardware [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1003.610628] env[64020]: DEBUG nova.virt.hardware [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1003.610825] env[64020]: DEBUG nova.virt.hardware [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1003.610971] env[64020]: DEBUG nova.virt.hardware [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1003.611127] env[64020]: DEBUG nova.virt.hardware [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1003.611301] env[64020]: DEBUG nova.virt.hardware [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1003.611483] env[64020]: DEBUG nova.virt.hardware [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1003.613018] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ef1b261-d000-43fb-9f2c-ab6466ea3052 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1003.622339] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5de21e4b-dece-4864-a057-014694fa38f1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1003.635725] env[64020]: ERROR nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 23ff4de9-4b22-4c53-afb4-ebdf8d8167b4, please check neutron logs for more information. [ 1003.635725] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Traceback (most recent call last): [ 1003.635725] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1003.635725] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] yield resources [ 1003.635725] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1003.635725] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] self.driver.spawn(context, instance, image_meta, [ 1003.635725] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1003.635725] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1003.635725] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1003.635725] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] vm_ref = self.build_virtual_machine(instance, [ 1003.635725] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] vif_infos = vmwarevif.get_vif_info(self._session, [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] for vif in network_info: [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] return self._sync_wrapper(fn, *args, **kwargs) [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] self.wait() [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] self[:] = self._gt.wait() [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] return self._exit_event.wait() [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1003.636099] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] current.throw(*self._exc) [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] result = function(*args, **kwargs) [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] return func(*args, **kwargs) [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] raise e [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] nwinfo = self.network_api.allocate_for_instance( [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] created_port_ids = self._update_ports_for_instance( [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] with excutils.save_and_reraise_exception(): [ 1003.636542] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1003.636939] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] self.force_reraise() [ 1003.636939] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1003.636939] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] raise self.value [ 1003.636939] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1003.636939] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] updated_port = self._update_port( [ 1003.636939] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1003.636939] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] _ensure_no_port_binding_failure(port) [ 1003.636939] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1003.636939] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] raise exception.PortBindingFailed(port_id=port['id']) [ 1003.636939] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] nova.exception.PortBindingFailed: Binding failed for port 23ff4de9-4b22-4c53-afb4-ebdf8d8167b4, please check neutron logs for more information. [ 1003.636939] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] [ 1003.636939] env[64020]: INFO nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Terminating instance [ 1003.638105] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Acquiring lock "refresh_cache-8f90d1e0-6ede-4e89-8423-170c0d0e605f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1003.775146] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.209s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1003.775626] env[64020]: DEBUG nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1003.777431] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 773dfcc5ae5a4ec3a55b7953edb27060 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1003.779038] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 6.954s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1003.781208] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg feb75eb8d2574c44a899b8e3fb382e0b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1003.808560] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 773dfcc5ae5a4ec3a55b7953edb27060 [ 1003.809806] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg feb75eb8d2574c44a899b8e3fb382e0b [ 1003.995017] env[64020]: DEBUG nova.network.neutron [req-5557a85d-4a19-4b5c-ae76-559768a7b2e7 req-70ca18c0-26dc-4ac8-8b1b-00846d592f6e service nova] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1004.102394] env[64020]: DEBUG nova.network.neutron [req-5557a85d-4a19-4b5c-ae76-559768a7b2e7 req-70ca18c0-26dc-4ac8-8b1b-00846d592f6e service nova] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1004.102917] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-5557a85d-4a19-4b5c-ae76-559768a7b2e7 req-70ca18c0-26dc-4ac8-8b1b-00846d592f6e service nova] Expecting reply to msg af65924239fb4457a57e9014c845fa2b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1004.111287] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af65924239fb4457a57e9014c845fa2b [ 1004.282956] env[64020]: DEBUG nova.compute.utils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1004.282956] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 72da390983d4458b88120577e33c6533 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1004.283827] env[64020]: DEBUG nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1004.283932] env[64020]: DEBUG nova.network.neutron [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1004.294708] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72da390983d4458b88120577e33c6533 [ 1004.343286] env[64020]: DEBUG nova.policy [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e9a022c35a624ebab962014dc0c72037', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8a8ad74a9f874a21838d224fa7638717', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1004.367955] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9bc1042-e663-4a92-947a-394d7a81c4df {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1004.375169] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ecf6618-2ae3-4851-8aac-ca7f20ec7811 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1004.405020] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e902284a-e132-4b15-91e9-9c363ed6d567 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1004.412091] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07a64aea-8523-4a45-9c76-3199cb63bf3f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1004.424759] env[64020]: DEBUG nova.compute.provider_tree [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1004.425240] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg ec1cd0df70cb424e8a75bbf7a322d3ca in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1004.432561] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec1cd0df70cb424e8a75bbf7a322d3ca [ 1004.604139] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "ed9362fc-3388-4c89-849b-8ef93d226331" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1004.604380] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "ed9362fc-3388-4c89-849b-8ef93d226331" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1004.604835] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 031889ae97e34c708823470dc876267e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1004.605713] env[64020]: DEBUG oslo_concurrency.lockutils [req-5557a85d-4a19-4b5c-ae76-559768a7b2e7 req-70ca18c0-26dc-4ac8-8b1b-00846d592f6e service nova] Releasing lock "refresh_cache-8f90d1e0-6ede-4e89-8423-170c0d0e605f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1004.605934] env[64020]: DEBUG nova.compute.manager [req-5557a85d-4a19-4b5c-ae76-559768a7b2e7 req-70ca18c0-26dc-4ac8-8b1b-00846d592f6e service nova] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Received event network-vif-deleted-23ff4de9-4b22-4c53-afb4-ebdf8d8167b4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1004.606409] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Acquired lock "refresh_cache-8f90d1e0-6ede-4e89-8423-170c0d0e605f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1004.606622] env[64020]: DEBUG nova.network.neutron [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1004.606993] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 39d885ef313f46c4970c4eefdf4c1e77 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1004.617498] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 39d885ef313f46c4970c4eefdf4c1e77 [ 1004.617964] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 031889ae97e34c708823470dc876267e [ 1004.654037] env[64020]: DEBUG nova.network.neutron [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Successfully created port: e184a494-6d9e-409c-bb63-61452c84516a {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1004.784345] env[64020]: DEBUG nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1004.786161] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg e3f2b8b986a14936b5ef8fcbd79d43b9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1004.819523] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3f2b8b986a14936b5ef8fcbd79d43b9 [ 1004.927320] env[64020]: DEBUG nova.scheduler.client.report [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1004.929841] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg cba40de4bdf540cb8cf254eef3e53f16 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1004.940143] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cba40de4bdf540cb8cf254eef3e53f16 [ 1005.107179] env[64020]: DEBUG nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1005.110161] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg dc492fa8a9754836992838758284dbb4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1005.140500] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc492fa8a9754836992838758284dbb4 [ 1005.193119] env[64020]: DEBUG nova.network.neutron [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1005.291978] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg eb86e5cbe40345eebf1745a7c33429fd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1005.321554] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb86e5cbe40345eebf1745a7c33429fd [ 1005.324455] env[64020]: DEBUG nova.network.neutron [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1005.325115] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg f879846870de4a2dacdfdb63c8006779 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1005.333454] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f879846870de4a2dacdfdb63c8006779 [ 1005.432271] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.653s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1005.432898] env[64020]: ERROR nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3cd0eb57-3579-49ff-8787-df8d6a4eadab, please check neutron logs for more information. [ 1005.432898] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] Traceback (most recent call last): [ 1005.432898] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1005.432898] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] self.driver.spawn(context, instance, image_meta, [ 1005.432898] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1005.432898] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1005.432898] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1005.432898] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] vm_ref = self.build_virtual_machine(instance, [ 1005.432898] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1005.432898] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] vif_infos = vmwarevif.get_vif_info(self._session, [ 1005.432898] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] for vif in network_info: [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] return self._sync_wrapper(fn, *args, **kwargs) [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] self.wait() [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] self[:] = self._gt.wait() [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] return self._exit_event.wait() [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] current.throw(*self._exc) [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1005.433283] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] result = function(*args, **kwargs) [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] return func(*args, **kwargs) [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] raise e [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] nwinfo = self.network_api.allocate_for_instance( [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] created_port_ids = self._update_ports_for_instance( [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] with excutils.save_and_reraise_exception(): [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] self.force_reraise() [ 1005.433700] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1005.434108] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] raise self.value [ 1005.434108] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1005.434108] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] updated_port = self._update_port( [ 1005.434108] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1005.434108] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] _ensure_no_port_binding_failure(port) [ 1005.434108] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1005.434108] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] raise exception.PortBindingFailed(port_id=port['id']) [ 1005.434108] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] nova.exception.PortBindingFailed: Binding failed for port 3cd0eb57-3579-49ff-8787-df8d6a4eadab, please check neutron logs for more information. [ 1005.434108] env[64020]: ERROR nova.compute.manager [instance: f616a54a-647c-4454-967c-e0ef50552720] [ 1005.434108] env[64020]: DEBUG nova.compute.utils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Binding failed for port 3cd0eb57-3579-49ff-8787-df8d6a4eadab, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1005.434819] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.214s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1005.436604] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg d3de11007c574e2c9995622a31389132 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1005.437739] env[64020]: DEBUG nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Build of instance f616a54a-647c-4454-967c-e0ef50552720 was re-scheduled: Binding failed for port 3cd0eb57-3579-49ff-8787-df8d6a4eadab, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1005.438167] env[64020]: DEBUG nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1005.438390] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquiring lock "refresh_cache-f616a54a-647c-4454-967c-e0ef50552720" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1005.438537] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Acquired lock "refresh_cache-f616a54a-647c-4454-967c-e0ef50552720" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1005.438692] env[64020]: DEBUG nova.network.neutron [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1005.439146] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 5ae148c173204dc3b518d8fc604505fe in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1005.445175] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ae148c173204dc3b518d8fc604505fe [ 1005.471411] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d3de11007c574e2c9995622a31389132 [ 1005.559025] env[64020]: DEBUG nova.compute.manager [req-a7ab3d4b-f73e-4cb0-9d79-aa0fa620414e req-75663999-59a1-4118-a4fb-b755d0d611e7 service nova] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Received event network-changed-e184a494-6d9e-409c-bb63-61452c84516a {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1005.562958] env[64020]: DEBUG nova.compute.manager [req-a7ab3d4b-f73e-4cb0-9d79-aa0fa620414e req-75663999-59a1-4118-a4fb-b755d0d611e7 service nova] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Refreshing instance network info cache due to event network-changed-e184a494-6d9e-409c-bb63-61452c84516a. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1005.562958] env[64020]: DEBUG oslo_concurrency.lockutils [req-a7ab3d4b-f73e-4cb0-9d79-aa0fa620414e req-75663999-59a1-4118-a4fb-b755d0d611e7 service nova] Acquiring lock "refresh_cache-d585fc50-c5a6-47bc-8672-498ac81c88f1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1005.562958] env[64020]: DEBUG oslo_concurrency.lockutils [req-a7ab3d4b-f73e-4cb0-9d79-aa0fa620414e req-75663999-59a1-4118-a4fb-b755d0d611e7 service nova] Acquired lock "refresh_cache-d585fc50-c5a6-47bc-8672-498ac81c88f1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1005.562958] env[64020]: DEBUG nova.network.neutron [req-a7ab3d4b-f73e-4cb0-9d79-aa0fa620414e req-75663999-59a1-4118-a4fb-b755d0d611e7 service nova] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Refreshing network info cache for port e184a494-6d9e-409c-bb63-61452c84516a {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1005.562958] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-a7ab3d4b-f73e-4cb0-9d79-aa0fa620414e req-75663999-59a1-4118-a4fb-b755d0d611e7 service nova] Expecting reply to msg 0e2ba9fe7800437da5af0bb8c8f8d3ae in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1005.567125] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e2ba9fe7800437da5af0bb8c8f8d3ae [ 1005.632184] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1005.709394] env[64020]: ERROR nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e184a494-6d9e-409c-bb63-61452c84516a, please check neutron logs for more information. [ 1005.709394] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1005.709394] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1005.709394] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1005.709394] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1005.709394] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1005.709394] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1005.709394] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1005.709394] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1005.709394] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1005.709394] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1005.709394] env[64020]: ERROR nova.compute.manager raise self.value [ 1005.709394] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1005.709394] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1005.709394] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1005.709394] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1005.709954] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1005.709954] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1005.709954] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e184a494-6d9e-409c-bb63-61452c84516a, please check neutron logs for more information. [ 1005.709954] env[64020]: ERROR nova.compute.manager [ 1005.709954] env[64020]: Traceback (most recent call last): [ 1005.709954] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1005.709954] env[64020]: listener.cb(fileno) [ 1005.709954] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1005.709954] env[64020]: result = function(*args, **kwargs) [ 1005.709954] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1005.709954] env[64020]: return func(*args, **kwargs) [ 1005.709954] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1005.709954] env[64020]: raise e [ 1005.709954] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1005.709954] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1005.709954] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1005.709954] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1005.709954] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1005.709954] env[64020]: with excutils.save_and_reraise_exception(): [ 1005.709954] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1005.709954] env[64020]: self.force_reraise() [ 1005.709954] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1005.709954] env[64020]: raise self.value [ 1005.709954] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1005.709954] env[64020]: updated_port = self._update_port( [ 1005.709954] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1005.709954] env[64020]: _ensure_no_port_binding_failure(port) [ 1005.709954] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1005.709954] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1005.710882] env[64020]: nova.exception.PortBindingFailed: Binding failed for port e184a494-6d9e-409c-bb63-61452c84516a, please check neutron logs for more information. [ 1005.710882] env[64020]: Removing descriptor: 17 [ 1005.795033] env[64020]: DEBUG nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1005.818898] env[64020]: DEBUG nova.virt.hardware [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1005.819150] env[64020]: DEBUG nova.virt.hardware [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1005.819439] env[64020]: DEBUG nova.virt.hardware [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1005.819695] env[64020]: DEBUG nova.virt.hardware [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1005.819850] env[64020]: DEBUG nova.virt.hardware [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1005.820012] env[64020]: DEBUG nova.virt.hardware [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1005.820234] env[64020]: DEBUG nova.virt.hardware [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1005.820393] env[64020]: DEBUG nova.virt.hardware [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1005.820558] env[64020]: DEBUG nova.virt.hardware [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1005.820719] env[64020]: DEBUG nova.virt.hardware [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1005.820886] env[64020]: DEBUG nova.virt.hardware [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1005.822064] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-046b1d21-aed8-494b-9089-12d1e0be091d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1005.826565] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Releasing lock "refresh_cache-8f90d1e0-6ede-4e89-8423-170c0d0e605f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1005.826925] env[64020]: DEBUG nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1005.827109] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1005.829507] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7b5b853d-d8b6-4596-8e8f-f94ca1099bad {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1005.832078] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-23f658c3-ce24-4c17-bacb-61ab65651c26 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1005.845100] env[64020]: ERROR nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e184a494-6d9e-409c-bb63-61452c84516a, please check neutron logs for more information. [ 1005.845100] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Traceback (most recent call last): [ 1005.845100] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1005.845100] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] yield resources [ 1005.845100] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1005.845100] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] self.driver.spawn(context, instance, image_meta, [ 1005.845100] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1005.845100] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1005.845100] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1005.845100] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] vm_ref = self.build_virtual_machine(instance, [ 1005.845100] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] vif_infos = vmwarevif.get_vif_info(self._session, [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] for vif in network_info: [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] return self._sync_wrapper(fn, *args, **kwargs) [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] self.wait() [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] self[:] = self._gt.wait() [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] return self._exit_event.wait() [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1005.845517] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] current.throw(*self._exc) [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] result = function(*args, **kwargs) [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] return func(*args, **kwargs) [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] raise e [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] nwinfo = self.network_api.allocate_for_instance( [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] created_port_ids = self._update_ports_for_instance( [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] with excutils.save_and_reraise_exception(): [ 1005.845940] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1005.846352] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] self.force_reraise() [ 1005.846352] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1005.846352] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] raise self.value [ 1005.846352] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1005.846352] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] updated_port = self._update_port( [ 1005.846352] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1005.846352] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] _ensure_no_port_binding_failure(port) [ 1005.846352] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1005.846352] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] raise exception.PortBindingFailed(port_id=port['id']) [ 1005.846352] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] nova.exception.PortBindingFailed: Binding failed for port e184a494-6d9e-409c-bb63-61452c84516a, please check neutron logs for more information. [ 1005.846352] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] [ 1005.846352] env[64020]: INFO nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Terminating instance [ 1005.847636] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "refresh_cache-d585fc50-c5a6-47bc-8672-498ac81c88f1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1005.850285] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a760c62b-be20-4c64-9dce-a2e4f1a721db {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1005.870503] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 8f90d1e0-6ede-4e89-8423-170c0d0e605f could not be found. [ 1005.870712] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1005.870882] env[64020]: INFO nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1005.871108] env[64020]: DEBUG oslo.service.loopingcall [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1005.871483] env[64020]: DEBUG nova.compute.manager [-] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1005.871595] env[64020]: DEBUG nova.network.neutron [-] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1005.890214] env[64020]: DEBUG nova.network.neutron [-] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1005.890957] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7cdbd7cd0a744724b5b897217af0d94e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1005.897614] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7cdbd7cd0a744724b5b897217af0d94e [ 1005.962025] env[64020]: DEBUG nova.network.neutron [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1006.019641] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00f328fc-897f-48bd-8532-021fde2f6f31 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1006.026632] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8aa93fc2-940b-48d4-ab30-493d89bae7e6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1006.055447] env[64020]: DEBUG nova.network.neutron [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1006.055937] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 70798c087a1f4c9695cb419306e24597 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1006.057425] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b379476-e71f-40fa-9259-9f0c41d10ca4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1006.065479] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 70798c087a1f4c9695cb419306e24597 [ 1006.066754] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d5c2497-cef9-439b-9800-a6d62e561b21 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1006.079960] env[64020]: DEBUG nova.compute.provider_tree [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1006.080401] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg d895cec80b8c48019e5d0c80e28b3925 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1006.083258] env[64020]: DEBUG nova.network.neutron [req-a7ab3d4b-f73e-4cb0-9d79-aa0fa620414e req-75663999-59a1-4118-a4fb-b755d0d611e7 service nova] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1006.086811] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d895cec80b8c48019e5d0c80e28b3925 [ 1006.156788] env[64020]: DEBUG nova.network.neutron [req-a7ab3d4b-f73e-4cb0-9d79-aa0fa620414e req-75663999-59a1-4118-a4fb-b755d0d611e7 service nova] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1006.157294] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-a7ab3d4b-f73e-4cb0-9d79-aa0fa620414e req-75663999-59a1-4118-a4fb-b755d0d611e7 service nova] Expecting reply to msg 5a8108ca4ff44e95bc4a4732545a0e83 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1006.164457] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a8108ca4ff44e95bc4a4732545a0e83 [ 1006.392741] env[64020]: DEBUG nova.network.neutron [-] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1006.393208] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 2f21f7d7048048d59b7c26a38e2cb5c6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1006.401105] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f21f7d7048048d59b7c26a38e2cb5c6 [ 1006.560905] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Releasing lock "refresh_cache-f616a54a-647c-4454-967c-e0ef50552720" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1006.561093] env[64020]: DEBUG nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1006.561432] env[64020]: DEBUG nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1006.561663] env[64020]: DEBUG nova.network.neutron [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1006.575888] env[64020]: DEBUG nova.network.neutron [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1006.576447] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 41dafdf6bf7a4479b673499f43f5c848 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1006.582294] env[64020]: DEBUG nova.scheduler.client.report [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1006.584579] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg f55a88969fed4c4aaf93b1e61c1a651a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1006.585512] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 41dafdf6bf7a4479b673499f43f5c848 [ 1006.594599] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f55a88969fed4c4aaf93b1e61c1a651a [ 1006.623392] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1006.623718] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1006.623894] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Starting heal instance info cache {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1006.623967] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Rebuilding the list of instances to heal {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1006.624504] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 027873b2484b4eb5b16ed24ab0d429b8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1006.635576] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 027873b2484b4eb5b16ed24ab0d429b8 [ 1006.659543] env[64020]: DEBUG oslo_concurrency.lockutils [req-a7ab3d4b-f73e-4cb0-9d79-aa0fa620414e req-75663999-59a1-4118-a4fb-b755d0d611e7 service nova] Releasing lock "refresh_cache-d585fc50-c5a6-47bc-8672-498ac81c88f1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1006.659907] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquired lock "refresh_cache-d585fc50-c5a6-47bc-8672-498ac81c88f1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1006.660103] env[64020]: DEBUG nova.network.neutron [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1006.660549] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 3c8253d43d9241b79a29f96047f11f46 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1006.666385] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c8253d43d9241b79a29f96047f11f46 [ 1006.895660] env[64020]: INFO nova.compute.manager [-] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Took 1.02 seconds to deallocate network for instance. [ 1006.898001] env[64020]: DEBUG nova.compute.claims [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1006.898164] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1007.078470] env[64020]: DEBUG nova.network.neutron [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1007.079016] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 1c6ce0a67f7c46bca0cbb88260c91b30 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1007.086933] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.652s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1007.087618] env[64020]: ERROR nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c83da57e-a466-44b2-89db-039f682c7fc1, please check neutron logs for more information. [ 1007.087618] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Traceback (most recent call last): [ 1007.087618] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1007.087618] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] self.driver.spawn(context, instance, image_meta, [ 1007.087618] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1007.087618] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1007.087618] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1007.087618] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] vm_ref = self.build_virtual_machine(instance, [ 1007.087618] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1007.087618] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] vif_infos = vmwarevif.get_vif_info(self._session, [ 1007.087618] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] for vif in network_info: [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] return self._sync_wrapper(fn, *args, **kwargs) [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] self.wait() [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] self[:] = self._gt.wait() [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] return self._exit_event.wait() [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] result = hub.switch() [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1007.088026] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] return self.greenlet.switch() [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] result = function(*args, **kwargs) [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] return func(*args, **kwargs) [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] raise e [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] nwinfo = self.network_api.allocate_for_instance( [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] created_port_ids = self._update_ports_for_instance( [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] with excutils.save_and_reraise_exception(): [ 1007.088466] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1007.088887] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] self.force_reraise() [ 1007.088887] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1007.088887] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] raise self.value [ 1007.088887] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1007.088887] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] updated_port = self._update_port( [ 1007.088887] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1007.088887] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] _ensure_no_port_binding_failure(port) [ 1007.088887] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1007.088887] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] raise exception.PortBindingFailed(port_id=port['id']) [ 1007.088887] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] nova.exception.PortBindingFailed: Binding failed for port c83da57e-a466-44b2-89db-039f682c7fc1, please check neutron logs for more information. [ 1007.088887] env[64020]: ERROR nova.compute.manager [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] [ 1007.089313] env[64020]: DEBUG nova.compute.utils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Binding failed for port c83da57e-a466-44b2-89db-039f682c7fc1, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1007.089678] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1c6ce0a67f7c46bca0cbb88260c91b30 [ 1007.091774] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 4.332s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1007.093613] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 0d0cbe70e9e0492791895eb257e8837d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1007.094602] env[64020]: DEBUG nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Build of instance bd872d96-d664-4ecb-ad59-516d95ad0cb7 was re-scheduled: Binding failed for port c83da57e-a466-44b2-89db-039f682c7fc1, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1007.095021] env[64020]: DEBUG nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1007.095240] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-bd872d96-d664-4ecb-ad59-516d95ad0cb7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1007.095388] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-bd872d96-d664-4ecb-ad59-516d95ad0cb7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1007.095540] env[64020]: DEBUG nova.network.neutron [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1007.095889] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg acea06262c024886a834913e1f5cb6fc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1007.101013] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg acea06262c024886a834913e1f5cb6fc [ 1007.127509] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1007.127645] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1007.127771] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1007.127893] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Didn't find any instances for network info cache update. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1007.128090] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1007.128298] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1007.128452] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1007.128601] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1007.128831] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1007.129047] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1007.129258] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=64020) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1007.129412] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager.update_available_resource {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1007.129824] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 1acacdf975b24dfb8a24de9f471f3658 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1007.138895] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0d0cbe70e9e0492791895eb257e8837d [ 1007.141797] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1acacdf975b24dfb8a24de9f471f3658 [ 1007.176475] env[64020]: DEBUG nova.network.neutron [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1007.242291] env[64020]: DEBUG nova.network.neutron [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1007.242799] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 8ec08295b2374193be9ba15cee066c5d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1007.250264] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ec08295b2374193be9ba15cee066c5d [ 1007.581979] env[64020]: INFO nova.compute.manager [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] [instance: f616a54a-647c-4454-967c-e0ef50552720] Took 1.02 seconds to deallocate network for instance. [ 1007.583479] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 08b1d575c158470a8c197c6e2e9a2930 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1007.585957] env[64020]: DEBUG nova.compute.manager [req-e2f36df9-21f7-4e5e-83ea-2636866c7817 req-b5e61fd7-9466-4284-b02a-f6c9216c0113 service nova] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Received event network-vif-deleted-e184a494-6d9e-409c-bb63-61452c84516a {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1007.616601] env[64020]: DEBUG nova.network.neutron [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1007.621133] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08b1d575c158470a8c197c6e2e9a2930 [ 1007.634002] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1007.687349] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0922f089-4321-4cde-aa7e-beca7fd7604a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1007.691937] env[64020]: DEBUG nova.network.neutron [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1007.692439] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 44f14f543c834b489376d390b3f4704d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1007.695675] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a094407c-49d5-4cd6-a906-74fa58572156 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1007.701671] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44f14f543c834b489376d390b3f4704d [ 1007.724831] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77b4f827-b234-479c-a30c-53a92a672e9c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1007.731353] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd384e65-1715-463e-b204-088627df6f32 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1007.745596] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Releasing lock "refresh_cache-d585fc50-c5a6-47bc-8672-498ac81c88f1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1007.746084] env[64020]: DEBUG nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1007.746321] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1007.746816] env[64020]: DEBUG nova.compute.provider_tree [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1007.747341] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 9238ee5168244c04a2b1f0c8df6ef389 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1007.748273] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4b272a51-32ec-4ea7-b851-92211bef53ed {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1007.756582] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89292bb7-7495-47a4-be47-310770aac186 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1007.770641] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9238ee5168244c04a2b1f0c8df6ef389 [ 1007.782722] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d585fc50-c5a6-47bc-8672-498ac81c88f1 could not be found. [ 1007.783083] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1007.783353] env[64020]: INFO nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1007.783733] env[64020]: DEBUG oslo.service.loopingcall [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1007.784062] env[64020]: DEBUG nova.compute.manager [-] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1007.784275] env[64020]: DEBUG nova.network.neutron [-] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1007.817567] env[64020]: DEBUG nova.network.neutron [-] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1007.818188] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a1babe83b8174450a1496037a36dad27 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1007.824509] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1babe83b8174450a1496037a36dad27 [ 1008.089865] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 47355ae47db4460b9e85ea47395aa392 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1008.120510] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47355ae47db4460b9e85ea47395aa392 [ 1008.196332] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-bd872d96-d664-4ecb-ad59-516d95ad0cb7" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1008.196669] env[64020]: DEBUG nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1008.196903] env[64020]: DEBUG nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1008.196991] env[64020]: DEBUG nova.network.neutron [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1008.210775] env[64020]: DEBUG nova.network.neutron [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1008.211329] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 02e11916fab64ec7b1e9a0977e918af6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1008.217420] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02e11916fab64ec7b1e9a0977e918af6 [ 1008.251758] env[64020]: DEBUG nova.scheduler.client.report [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1008.254017] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 97a69fdf0e8d48438230bbbb84a9f856 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1008.264172] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 97a69fdf0e8d48438230bbbb84a9f856 [ 1008.320456] env[64020]: DEBUG nova.network.neutron [-] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1008.320914] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 93983db6a3ea49be84ca7fc17647f43f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1008.329040] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93983db6a3ea49be84ca7fc17647f43f [ 1008.610528] env[64020]: INFO nova.scheduler.client.report [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Deleted allocations for instance f616a54a-647c-4454-967c-e0ef50552720 [ 1008.621586] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Expecting reply to msg 30ec50b0ac91437f8ba64cad4b03bc3e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1008.632065] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30ec50b0ac91437f8ba64cad4b03bc3e [ 1008.713083] env[64020]: DEBUG nova.network.neutron [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1008.713612] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg fea740fa00e74ea09b0c09a476d68793 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1008.721988] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fea740fa00e74ea09b0c09a476d68793 [ 1008.755823] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.664s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1008.756428] env[64020]: ERROR nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 63fa9007-0069-463c-a79b-3bd3c6dcb3ce, please check neutron logs for more information. [ 1008.756428] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Traceback (most recent call last): [ 1008.756428] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1008.756428] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] self.driver.spawn(context, instance, image_meta, [ 1008.756428] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1008.756428] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1008.756428] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1008.756428] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] vm_ref = self.build_virtual_machine(instance, [ 1008.756428] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1008.756428] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] vif_infos = vmwarevif.get_vif_info(self._session, [ 1008.756428] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] for vif in network_info: [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] return self._sync_wrapper(fn, *args, **kwargs) [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] self.wait() [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] self[:] = self._gt.wait() [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] return self._exit_event.wait() [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] current.throw(*self._exc) [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1008.756797] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] result = function(*args, **kwargs) [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] return func(*args, **kwargs) [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] raise e [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] nwinfo = self.network_api.allocate_for_instance( [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] created_port_ids = self._update_ports_for_instance( [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] with excutils.save_and_reraise_exception(): [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] self.force_reraise() [ 1008.757159] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1008.757520] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] raise self.value [ 1008.757520] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1008.757520] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] updated_port = self._update_port( [ 1008.757520] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1008.757520] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] _ensure_no_port_binding_failure(port) [ 1008.757520] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1008.757520] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] raise exception.PortBindingFailed(port_id=port['id']) [ 1008.757520] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] nova.exception.PortBindingFailed: Binding failed for port 63fa9007-0069-463c-a79b-3bd3c6dcb3ce, please check neutron logs for more information. [ 1008.757520] env[64020]: ERROR nova.compute.manager [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] [ 1008.757520] env[64020]: DEBUG nova.compute.utils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Binding failed for port 63fa9007-0069-463c-a79b-3bd3c6dcb3ce, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1008.758884] env[64020]: DEBUG nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Build of instance 6db545fe-881e-4c9a-b514-bd98d7528674 was re-scheduled: Binding failed for port 63fa9007-0069-463c-a79b-3bd3c6dcb3ce, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1008.759275] env[64020]: DEBUG nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1008.759502] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-6db545fe-881e-4c9a-b514-bd98d7528674" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1008.759648] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-6db545fe-881e-4c9a-b514-bd98d7528674" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1008.759805] env[64020]: DEBUG nova.network.neutron [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1008.760240] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg a2af3c2707d54d6bac9c5419d12db4c6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1008.761406] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.129s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1008.763047] env[64020]: INFO nova.compute.claims [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1008.765139] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg d89676e590934975994f4a804b416531 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1008.766908] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2af3c2707d54d6bac9c5419d12db4c6 [ 1008.797188] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d89676e590934975994f4a804b416531 [ 1008.823246] env[64020]: INFO nova.compute.manager [-] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Took 1.04 seconds to deallocate network for instance. [ 1008.825525] env[64020]: DEBUG nova.compute.claims [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1008.825734] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1009.118383] env[64020]: DEBUG oslo_concurrency.lockutils [None req-4037a146-0ac4-492a-ab4e-c2faac738388 tempest-SecurityGroupsTestJSON-586403488 tempest-SecurityGroupsTestJSON-586403488-project-member] Lock "f616a54a-647c-4454-967c-e0ef50552720" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 39.179s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1009.216453] env[64020]: INFO nova.compute.manager [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bd872d96-d664-4ecb-ad59-516d95ad0cb7] Took 1.02 seconds to deallocate network for instance. [ 1009.217951] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 0a44f7f9af9b49109f1dbd7d3f673e37 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1009.255354] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a44f7f9af9b49109f1dbd7d3f673e37 [ 1009.268357] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg d7f03be2336d4bd3be70199fca9685ab in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1009.278982] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7f03be2336d4bd3be70199fca9685ab [ 1009.278982] env[64020]: DEBUG nova.network.neutron [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1009.381766] env[64020]: DEBUG nova.network.neutron [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1009.382282] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg d297d2f731c4456388c5596d715cbe42 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1009.390576] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d297d2f731c4456388c5596d715cbe42 [ 1009.723499] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 4d6b0b4f04c84eb9b425edd5e8a60579 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1009.754470] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d6b0b4f04c84eb9b425edd5e8a60579 [ 1009.841701] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2326fe21-0a01-4504-b3d4-308d08967392 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1009.849000] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9acfc7d0-90a9-4eea-891f-83ccbc85891a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1009.878268] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7bd53f86-fa62-45cd-87ee-1bc396fa3be2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1009.884445] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-6db545fe-881e-4c9a-b514-bd98d7528674" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1009.884665] env[64020]: DEBUG nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1009.884845] env[64020]: DEBUG nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1009.885009] env[64020]: DEBUG nova.network.neutron [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1009.887702] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d14ce4de-d81f-4888-9108-465f002b3e21 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1009.900882] env[64020]: DEBUG nova.compute.provider_tree [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1009.901303] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg f7aa49ca52a24640b868ec1cb7db3f7e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1009.909431] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7aa49ca52a24640b868ec1cb7db3f7e [ 1009.910516] env[64020]: DEBUG nova.network.neutron [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1009.911047] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 5ef12c9b4a8647269e495e89fea15cf0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1009.918435] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ef12c9b4a8647269e495e89fea15cf0 [ 1010.244674] env[64020]: INFO nova.scheduler.client.report [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Deleted allocations for instance bd872d96-d664-4ecb-ad59-516d95ad0cb7 [ 1010.263083] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 23f2cb68997941098ace23166becd14f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1010.276118] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 23f2cb68997941098ace23166becd14f [ 1010.404239] env[64020]: DEBUG nova.scheduler.client.report [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1010.406584] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg eac707400c2046f1812ce0f800f43c51 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1010.413025] env[64020]: DEBUG nova.network.neutron [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1010.413509] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 8daa32a4b0204c5aa9da59ed6ef5b958 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1010.422143] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8daa32a4b0204c5aa9da59ed6ef5b958 [ 1010.424991] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eac707400c2046f1812ce0f800f43c51 [ 1010.769375] env[64020]: DEBUG oslo_concurrency.lockutils [None req-edf047f8-6fea-4813-a097-a0a4ee3cfef8 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "bd872d96-d664-4ecb-ad59-516d95ad0cb7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 31.978s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1010.910007] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.148s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1010.910556] env[64020]: DEBUG nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1010.912340] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg ad62f032ee104867be27af039c113ea7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1010.913346] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 4.015s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1010.915149] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 47d19c2fdcc44560bde89a4741ae90aa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1010.916212] env[64020]: INFO nova.compute.manager [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 6db545fe-881e-4c9a-b514-bd98d7528674] Took 1.03 seconds to deallocate network for instance. [ 1010.925734] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 853290069dd84b2a96d9b35ef29d7ca8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1010.953235] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47d19c2fdcc44560bde89a4741ae90aa [ 1010.968502] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 853290069dd84b2a96d9b35ef29d7ca8 [ 1010.977815] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad62f032ee104867be27af039c113ea7 [ 1011.428800] env[64020]: DEBUG nova.compute.utils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1011.429462] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg ec48f2b6883e46a4ad084915c129bf58 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1011.435345] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg d48f1a3127314d23aa59daecef668258 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1011.436629] env[64020]: DEBUG nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1011.437695] env[64020]: DEBUG nova.network.neutron [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1011.441106] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec48f2b6883e46a4ad084915c129bf58 [ 1011.473954] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d48f1a3127314d23aa59daecef668258 [ 1011.503191] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1fd7562-2ee5-4ff3-b8a8-9339480f826f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1011.507354] env[64020]: DEBUG nova.policy [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fd08d772c4dd4bf5986d37b8474d30a1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '540a31f288b54221a4d0348b147e40e7', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1011.511993] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-791b837c-49a6-4e8e-b0a9-1544fbd542d9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1011.545116] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a24724b-9900-4e84-b1d8-aa5edaecb8fa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1011.553186] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-414a8111-8bab-47f0-87ae-68d10cda7616 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1011.566808] env[64020]: DEBUG nova.compute.provider_tree [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1011.567333] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg ca30bc2ff15145c6b6a3f083b330d67d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1011.574608] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca30bc2ff15145c6b6a3f083b330d67d [ 1011.791905] env[64020]: DEBUG nova.network.neutron [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Successfully created port: 49f6eaa0-e982-4031-95c8-109122d3ba0d {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1011.928758] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "7ebc2bed-b2a4-4649-b873-66955c30b3f3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1011.928981] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "7ebc2bed-b2a4-4649-b873-66955c30b3f3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1011.929450] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 9581ba719c024627897129da37f3eb24 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1011.937291] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9581ba719c024627897129da37f3eb24 [ 1011.937846] env[64020]: DEBUG nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1011.939420] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg ba53e14298f8454789496e3bf600d019 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1011.957225] env[64020]: INFO nova.scheduler.client.report [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Deleted allocations for instance 6db545fe-881e-4c9a-b514-bd98d7528674 [ 1011.962784] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg b5b569398a294f54b5d22325b3479cde in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1011.972642] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b5b569398a294f54b5d22325b3479cde [ 1011.976605] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba53e14298f8454789496e3bf600d019 [ 1012.070141] env[64020]: DEBUG nova.scheduler.client.report [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1012.073159] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg f7179f63e1e2410989fd09f8b31845d7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1012.084915] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7179f63e1e2410989fd09f8b31845d7 [ 1012.391728] env[64020]: DEBUG nova.compute.manager [req-239d7b1e-166c-4671-9989-6f978ed7e389 req-736236be-9d84-4df8-83e1-23441300d56e service nova] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Received event network-changed-49f6eaa0-e982-4031-95c8-109122d3ba0d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1012.391991] env[64020]: DEBUG nova.compute.manager [req-239d7b1e-166c-4671-9989-6f978ed7e389 req-736236be-9d84-4df8-83e1-23441300d56e service nova] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Refreshing instance network info cache due to event network-changed-49f6eaa0-e982-4031-95c8-109122d3ba0d. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1012.392182] env[64020]: DEBUG oslo_concurrency.lockutils [req-239d7b1e-166c-4671-9989-6f978ed7e389 req-736236be-9d84-4df8-83e1-23441300d56e service nova] Acquiring lock "refresh_cache-ed9362fc-3388-4c89-849b-8ef93d226331" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1012.392311] env[64020]: DEBUG oslo_concurrency.lockutils [req-239d7b1e-166c-4671-9989-6f978ed7e389 req-736236be-9d84-4df8-83e1-23441300d56e service nova] Acquired lock "refresh_cache-ed9362fc-3388-4c89-849b-8ef93d226331" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1012.392456] env[64020]: DEBUG nova.network.neutron [req-239d7b1e-166c-4671-9989-6f978ed7e389 req-736236be-9d84-4df8-83e1-23441300d56e service nova] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Refreshing network info cache for port 49f6eaa0-e982-4031-95c8-109122d3ba0d {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1012.392872] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-239d7b1e-166c-4671-9989-6f978ed7e389 req-736236be-9d84-4df8-83e1-23441300d56e service nova] Expecting reply to msg b9483954737349e3b16e6441e8fe92e0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1012.399539] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9483954737349e3b16e6441e8fe92e0 [ 1012.431294] env[64020]: DEBUG nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1012.433490] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg fae20c594767417a9a2e36c0b7a2db23 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1012.451490] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 5f0607a9553e4a4a86cf336175801dd7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1012.462795] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fae20c594767417a9a2e36c0b7a2db23 [ 1012.464402] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a6df5365-fc46-4d73-93ea-1add5179414e tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "6db545fe-881e-4c9a-b514-bd98d7528674" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 30.715s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1012.482834] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5f0607a9553e4a4a86cf336175801dd7 [ 1012.576043] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.663s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1012.576678] env[64020]: ERROR nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 23ff4de9-4b22-4c53-afb4-ebdf8d8167b4, please check neutron logs for more information. [ 1012.576678] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Traceback (most recent call last): [ 1012.576678] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1012.576678] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] self.driver.spawn(context, instance, image_meta, [ 1012.576678] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1012.576678] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1012.576678] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1012.576678] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] vm_ref = self.build_virtual_machine(instance, [ 1012.576678] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1012.576678] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] vif_infos = vmwarevif.get_vif_info(self._session, [ 1012.576678] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] for vif in network_info: [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] return self._sync_wrapper(fn, *args, **kwargs) [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] self.wait() [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] self[:] = self._gt.wait() [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] return self._exit_event.wait() [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] current.throw(*self._exc) [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1012.577018] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] result = function(*args, **kwargs) [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] return func(*args, **kwargs) [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] raise e [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] nwinfo = self.network_api.allocate_for_instance( [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] created_port_ids = self._update_ports_for_instance( [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] with excutils.save_and_reraise_exception(): [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] self.force_reraise() [ 1012.577376] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1012.577768] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] raise self.value [ 1012.577768] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1012.577768] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] updated_port = self._update_port( [ 1012.577768] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1012.577768] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] _ensure_no_port_binding_failure(port) [ 1012.577768] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1012.577768] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] raise exception.PortBindingFailed(port_id=port['id']) [ 1012.577768] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] nova.exception.PortBindingFailed: Binding failed for port 23ff4de9-4b22-4c53-afb4-ebdf8d8167b4, please check neutron logs for more information. [ 1012.577768] env[64020]: ERROR nova.compute.manager [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] [ 1012.577768] env[64020]: DEBUG nova.compute.utils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Binding failed for port 23ff4de9-4b22-4c53-afb4-ebdf8d8167b4, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1012.578526] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 4.945s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1012.578696] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1012.578843] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=64020) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1012.579111] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 3.753s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1012.581009] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 060233867cbf43d691518bbb6b75c94b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1012.582050] env[64020]: DEBUG nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Build of instance 8f90d1e0-6ede-4e89-8423-170c0d0e605f was re-scheduled: Binding failed for port 23ff4de9-4b22-4c53-afb4-ebdf8d8167b4, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1012.582446] env[64020]: DEBUG nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1012.582728] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Acquiring lock "refresh_cache-8f90d1e0-6ede-4e89-8423-170c0d0e605f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1012.582878] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Acquired lock "refresh_cache-8f90d1e0-6ede-4e89-8423-170c0d0e605f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1012.583035] env[64020]: DEBUG nova.network.neutron [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1012.583380] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 8ce78bd8487a4a0abb60f74b106b00d8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1012.584563] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc955931-2876-45db-ae8b-2a77be067ad3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1012.589368] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ce78bd8487a4a0abb60f74b106b00d8 [ 1012.593948] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd66d3ad-14c0-452c-b517-de636699c971 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1012.600094] env[64020]: ERROR nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 49f6eaa0-e982-4031-95c8-109122d3ba0d, please check neutron logs for more information. [ 1012.600094] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1012.600094] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1012.600094] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1012.600094] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1012.600094] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1012.600094] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1012.600094] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1012.600094] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1012.600094] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1012.600094] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1012.600094] env[64020]: ERROR nova.compute.manager raise self.value [ 1012.600094] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1012.600094] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1012.600094] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1012.600094] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1012.600948] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1012.600948] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1012.600948] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 49f6eaa0-e982-4031-95c8-109122d3ba0d, please check neutron logs for more information. [ 1012.600948] env[64020]: ERROR nova.compute.manager [ 1012.600948] env[64020]: Traceback (most recent call last): [ 1012.600948] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1012.600948] env[64020]: listener.cb(fileno) [ 1012.600948] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1012.600948] env[64020]: result = function(*args, **kwargs) [ 1012.600948] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1012.600948] env[64020]: return func(*args, **kwargs) [ 1012.600948] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1012.600948] env[64020]: raise e [ 1012.600948] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1012.600948] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1012.600948] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1012.600948] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1012.600948] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1012.600948] env[64020]: with excutils.save_and_reraise_exception(): [ 1012.600948] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1012.600948] env[64020]: self.force_reraise() [ 1012.600948] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1012.600948] env[64020]: raise self.value [ 1012.600948] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1012.600948] env[64020]: updated_port = self._update_port( [ 1012.600948] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1012.600948] env[64020]: _ensure_no_port_binding_failure(port) [ 1012.600948] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1012.600948] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1012.602380] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 49f6eaa0-e982-4031-95c8-109122d3ba0d, please check neutron logs for more information. [ 1012.602380] env[64020]: Removing descriptor: 17 [ 1012.608903] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d4c72d0-4f61-4fcb-b61e-b1166807ee6a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1012.615473] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c390dc8e-ea36-4a44-9be2-88388ca3ff40 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1012.618786] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 060233867cbf43d691518bbb6b75c94b [ 1012.644145] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181776MB free_disk=120GB free_vcpus=48 pci_devices=None {{(pid=64020) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1012.644302] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1012.909322] env[64020]: DEBUG nova.network.neutron [req-239d7b1e-166c-4671-9989-6f978ed7e389 req-736236be-9d84-4df8-83e1-23441300d56e service nova] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1012.949665] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1012.953773] env[64020]: DEBUG nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1012.972946] env[64020]: DEBUG nova.virt.hardware [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1012.973174] env[64020]: DEBUG nova.virt.hardware [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1012.973329] env[64020]: DEBUG nova.virt.hardware [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1012.973504] env[64020]: DEBUG nova.virt.hardware [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1012.973644] env[64020]: DEBUG nova.virt.hardware [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1012.974238] env[64020]: DEBUG nova.virt.hardware [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1012.974238] env[64020]: DEBUG nova.virt.hardware [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1012.974238] env[64020]: DEBUG nova.virt.hardware [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1012.974374] env[64020]: DEBUG nova.virt.hardware [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1012.974415] env[64020]: DEBUG nova.virt.hardware [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1012.974613] env[64020]: DEBUG nova.virt.hardware [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1012.975380] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5af636fb-dd9d-4bd4-bd96-97f047e66ec8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1012.980360] env[64020]: DEBUG nova.network.neutron [req-239d7b1e-166c-4671-9989-6f978ed7e389 req-736236be-9d84-4df8-83e1-23441300d56e service nova] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1012.980816] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-239d7b1e-166c-4671-9989-6f978ed7e389 req-736236be-9d84-4df8-83e1-23441300d56e service nova] Expecting reply to msg f20b1fd210a74cf6923eeafb2dbeef75 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1012.987650] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-880f7f76-fe49-400c-8cee-62a4fc859f6c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1012.992266] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f20b1fd210a74cf6923eeafb2dbeef75 [ 1013.003143] env[64020]: ERROR nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 49f6eaa0-e982-4031-95c8-109122d3ba0d, please check neutron logs for more information. [ 1013.003143] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Traceback (most recent call last): [ 1013.003143] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1013.003143] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] yield resources [ 1013.003143] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1013.003143] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] self.driver.spawn(context, instance, image_meta, [ 1013.003143] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1013.003143] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1013.003143] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1013.003143] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] vm_ref = self.build_virtual_machine(instance, [ 1013.003143] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] vif_infos = vmwarevif.get_vif_info(self._session, [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] for vif in network_info: [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] return self._sync_wrapper(fn, *args, **kwargs) [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] self.wait() [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] self[:] = self._gt.wait() [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] return self._exit_event.wait() [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1013.003580] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] current.throw(*self._exc) [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] result = function(*args, **kwargs) [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] return func(*args, **kwargs) [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] raise e [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] nwinfo = self.network_api.allocate_for_instance( [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] created_port_ids = self._update_ports_for_instance( [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] with excutils.save_and_reraise_exception(): [ 1013.004036] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1013.004487] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] self.force_reraise() [ 1013.004487] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1013.004487] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] raise self.value [ 1013.004487] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1013.004487] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] updated_port = self._update_port( [ 1013.004487] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1013.004487] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] _ensure_no_port_binding_failure(port) [ 1013.004487] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1013.004487] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] raise exception.PortBindingFailed(port_id=port['id']) [ 1013.004487] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] nova.exception.PortBindingFailed: Binding failed for port 49f6eaa0-e982-4031-95c8-109122d3ba0d, please check neutron logs for more information. [ 1013.004487] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] [ 1013.004487] env[64020]: INFO nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Terminating instance [ 1013.005140] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "refresh_cache-ed9362fc-3388-4c89-849b-8ef93d226331" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1013.108265] env[64020]: DEBUG nova.network.neutron [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1013.170869] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d9d0d37-7b98-4647-b860-31700fce4c06 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.178548] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1840bf63-dbfa-4d1a-ab47-661086280f65 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.207000] env[64020]: DEBUG nova.network.neutron [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1013.207566] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg a64d6fc2bfc24edfbdf04914758577cb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1013.208862] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d71f950e-a47f-4512-b826-801f4f9e6363 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.215789] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08655af1-1273-475c-a166-a1abcb8424e6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.219665] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a64d6fc2bfc24edfbdf04914758577cb [ 1013.229233] env[64020]: DEBUG nova.compute.provider_tree [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1013.229726] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 28f4891293474a2ba67baff9a0a6efa1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1013.236448] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28f4891293474a2ba67baff9a0a6efa1 [ 1013.430125] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "11849b75-4478-4bb5-b8eb-bb03b5c4dd57" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.430230] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "11849b75-4478-4bb5-b8eb-bb03b5c4dd57" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1013.430670] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 94c1d4e98c394c279265b31f31e65672 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1013.440465] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94c1d4e98c394c279265b31f31e65672 [ 1013.487030] env[64020]: DEBUG oslo_concurrency.lockutils [req-239d7b1e-166c-4671-9989-6f978ed7e389 req-736236be-9d84-4df8-83e1-23441300d56e service nova] Releasing lock "refresh_cache-ed9362fc-3388-4c89-849b-8ef93d226331" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1013.487506] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquired lock "refresh_cache-ed9362fc-3388-4c89-849b-8ef93d226331" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1013.487717] env[64020]: DEBUG nova.network.neutron [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1013.488378] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 3344de605400412ab1d5dc353f84b595 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1013.495268] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3344de605400412ab1d5dc353f84b595 [ 1013.712024] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Releasing lock "refresh_cache-8f90d1e0-6ede-4e89-8423-170c0d0e605f" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1013.712304] env[64020]: DEBUG nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1013.712528] env[64020]: DEBUG nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1013.712699] env[64020]: DEBUG nova.network.neutron [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1013.732412] env[64020]: DEBUG nova.network.neutron [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1013.732969] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 48aff2fcd2734e8fb7de8dd14ea88d9e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1013.734366] env[64020]: DEBUG nova.scheduler.client.report [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1013.736710] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 88ae2114d5064a91a233689a46f82875 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1013.741242] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 48aff2fcd2734e8fb7de8dd14ea88d9e [ 1013.749535] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88ae2114d5064a91a233689a46f82875 [ 1013.935444] env[64020]: DEBUG nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1013.935444] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 403a20c135d24d4681dc56361ad2d00b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1013.964818] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 403a20c135d24d4681dc56361ad2d00b [ 1014.004146] env[64020]: DEBUG nova.network.neutron [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1014.075934] env[64020]: DEBUG nova.network.neutron [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1014.076553] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 4f443eafc87f4fba91bbac64d76b0c6c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1014.084581] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f443eafc87f4fba91bbac64d76b0c6c [ 1014.238841] env[64020]: DEBUG nova.network.neutron [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1014.239346] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg c1fb9ab41f2c49e68a3cd5bf9537e75c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1014.241135] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.662s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1014.241754] env[64020]: ERROR nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e184a494-6d9e-409c-bb63-61452c84516a, please check neutron logs for more information. [ 1014.241754] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Traceback (most recent call last): [ 1014.241754] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1014.241754] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] self.driver.spawn(context, instance, image_meta, [ 1014.241754] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1014.241754] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1014.241754] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1014.241754] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] vm_ref = self.build_virtual_machine(instance, [ 1014.241754] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1014.241754] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] vif_infos = vmwarevif.get_vif_info(self._session, [ 1014.241754] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] for vif in network_info: [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] return self._sync_wrapper(fn, *args, **kwargs) [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] self.wait() [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] self[:] = self._gt.wait() [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] return self._exit_event.wait() [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] current.throw(*self._exc) [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1014.242248] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] result = function(*args, **kwargs) [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] return func(*args, **kwargs) [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] raise e [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] nwinfo = self.network_api.allocate_for_instance( [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] created_port_ids = self._update_ports_for_instance( [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] with excutils.save_and_reraise_exception(): [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] self.force_reraise() [ 1014.242710] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1014.243155] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] raise self.value [ 1014.243155] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1014.243155] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] updated_port = self._update_port( [ 1014.243155] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1014.243155] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] _ensure_no_port_binding_failure(port) [ 1014.243155] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1014.243155] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] raise exception.PortBindingFailed(port_id=port['id']) [ 1014.243155] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] nova.exception.PortBindingFailed: Binding failed for port e184a494-6d9e-409c-bb63-61452c84516a, please check neutron logs for more information. [ 1014.243155] env[64020]: ERROR nova.compute.manager [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] [ 1014.243155] env[64020]: DEBUG nova.compute.utils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Binding failed for port e184a494-6d9e-409c-bb63-61452c84516a, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1014.243518] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 1.599s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1014.244442] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg df30e59eb9e741848a85727b8a29259f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1014.245672] env[64020]: DEBUG nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Build of instance d585fc50-c5a6-47bc-8672-498ac81c88f1 was re-scheduled: Binding failed for port e184a494-6d9e-409c-bb63-61452c84516a, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1014.246091] env[64020]: DEBUG nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1014.246313] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "refresh_cache-d585fc50-c5a6-47bc-8672-498ac81c88f1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1014.246461] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquired lock "refresh_cache-d585fc50-c5a6-47bc-8672-498ac81c88f1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1014.246617] env[64020]: DEBUG nova.network.neutron [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1014.246962] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 73148f18405441b1ac71c7408ccae16d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1014.251382] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c1fb9ab41f2c49e68a3cd5bf9537e75c [ 1014.252776] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 73148f18405441b1ac71c7408ccae16d [ 1014.255180] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg df30e59eb9e741848a85727b8a29259f [ 1014.417063] env[64020]: DEBUG nova.compute.manager [req-2338daa5-c46e-4520-aa73-4043b31d3347 req-a920185d-b9de-46dd-8460-66d238f0975e service nova] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Received event network-vif-deleted-49f6eaa0-e982-4031-95c8-109122d3ba0d {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1014.450834] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1014.579217] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Releasing lock "refresh_cache-ed9362fc-3388-4c89-849b-8ef93d226331" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1014.579725] env[64020]: DEBUG nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1014.579940] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1014.580402] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7f64f29f-3189-45fc-9ec8-65218e551a41 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1014.589154] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-989e9915-2dd0-438e-b9be-07fb242bd71e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1014.609621] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ed9362fc-3388-4c89-849b-8ef93d226331 could not be found. [ 1014.609822] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1014.609998] env[64020]: INFO nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1014.610223] env[64020]: DEBUG oslo.service.loopingcall [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1014.610434] env[64020]: DEBUG nova.compute.manager [-] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1014.610609] env[64020]: DEBUG nova.network.neutron [-] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1014.624152] env[64020]: DEBUG nova.network.neutron [-] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1014.624602] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg eff43cdb89a742e8aacd270e5d8728c4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1014.631103] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eff43cdb89a742e8aacd270e5d8728c4 [ 1014.741408] env[64020]: INFO nova.compute.manager [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] [instance: 8f90d1e0-6ede-4e89-8423-170c0d0e605f] Took 1.03 seconds to deallocate network for instance. [ 1014.743408] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 0583e4665a6a4c20b687124b76f056e3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1014.746781] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 439ff2c071d94bfdb65c27efe224cd6a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1014.758244] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 439ff2c071d94bfdb65c27efe224cd6a [ 1014.763041] env[64020]: DEBUG nova.network.neutron [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1014.777305] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0583e4665a6a4c20b687124b76f056e3 [ 1014.840925] env[64020]: DEBUG nova.network.neutron [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1014.841460] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 69dbf54ff0cf4ba3aebad9dc0364da0b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1014.848924] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69dbf54ff0cf4ba3aebad9dc0364da0b [ 1015.126250] env[64020]: DEBUG nova.network.neutron [-] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1015.126673] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b064bfe6d62b4abe8de873544ec6b43f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1015.134799] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b064bfe6d62b4abe8de873544ec6b43f [ 1015.249911] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg 6aefc72fd6db4ea980cb1a394299b159 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1015.269842] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 2893a9a58df54960a4b00eabca02b6e8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1015.281338] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6aefc72fd6db4ea980cb1a394299b159 [ 1015.282012] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2893a9a58df54960a4b00eabca02b6e8 [ 1015.343889] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Releasing lock "refresh_cache-d585fc50-c5a6-47bc-8672-498ac81c88f1" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1015.344146] env[64020]: DEBUG nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1015.344333] env[64020]: DEBUG nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1015.344502] env[64020]: DEBUG nova.network.neutron [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1015.359109] env[64020]: DEBUG nova.network.neutron [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1015.359660] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg e6cce1e379014ebc9dc11c04de33e7e6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1015.366113] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6cce1e379014ebc9dc11c04de33e7e6 [ 1016.060709] env[64020]: INFO nova.compute.manager [-] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Took 1.45 seconds to deallocate network for instance. [ 1016.063765] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 8f90d1e0-6ede-4e89-8423-170c0d0e605f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1016.064237] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg ef886ef8e7884ac793515229fc482644 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1016.065034] env[64020]: DEBUG nova.network.neutron [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1016.065419] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 677c26e636744904b26cbce06e01d132 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1016.067919] env[64020]: DEBUG nova.compute.claims [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1016.068099] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1016.073424] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 677c26e636744904b26cbce06e01d132 [ 1016.073976] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef886ef8e7884ac793515229fc482644 [ 1016.081470] env[64020]: INFO nova.scheduler.client.report [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Deleted allocations for instance 8f90d1e0-6ede-4e89-8423-170c0d0e605f [ 1016.086877] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Expecting reply to msg f3159d2b38314154af54ad73f13c4f74 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1016.097730] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3159d2b38314154af54ad73f13c4f74 [ 1016.567331] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance d585fc50-c5a6-47bc-8672-498ac81c88f1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1016.567576] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance ed9362fc-3388-4c89-849b-8ef93d226331 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1016.568127] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg a3ca091e79f3435d822c7ceaf1831393 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1016.569254] env[64020]: INFO nova.compute.manager [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: d585fc50-c5a6-47bc-8672-498ac81c88f1] Took 1.22 seconds to deallocate network for instance. [ 1016.570997] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 7b784516163345a6aadb712a68239b57 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1016.579147] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a3ca091e79f3435d822c7ceaf1831393 [ 1016.588699] env[64020]: DEBUG oslo_concurrency.lockutils [None req-7d40e1fc-400f-4491-9796-2c03b603777e tempest-ServerActionsTestJSON-347776761 tempest-ServerActionsTestJSON-347776761-project-member] Lock "8f90d1e0-6ede-4e89-8423-170c0d0e605f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 24.876s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1016.602867] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7b784516163345a6aadb712a68239b57 [ 1017.073156] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 7ebc2bed-b2a4-4649-b873-66955c30b3f3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1017.073730] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg b1654f9a3c3740c7842811aca5d94755 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1017.076763] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg b2e9be4c1d1d4fc48aa3c72e148a983e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1017.083367] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1654f9a3c3740c7842811aca5d94755 [ 1017.107022] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b2e9be4c1d1d4fc48aa3c72e148a983e [ 1017.578977] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 11849b75-4478-4bb5-b8eb-bb03b5c4dd57 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1017.579227] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Total usable vcpus: 48, total allocated vcpus: 1 {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1017.579384] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=704MB phys_disk=200GB used_disk=1GB total_vcpus=48 used_vcpus=1 pci_stats=[] {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1017.599375] env[64020]: INFO nova.scheduler.client.report [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Deleted allocations for instance d585fc50-c5a6-47bc-8672-498ac81c88f1 [ 1017.607040] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg bd947ee5a26a43a4bc6122c18f583c5f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1017.623335] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bd947ee5a26a43a4bc6122c18f583c5f [ 1017.639900] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-340ab190-7951-4e20-94ea-eb3ce43976a6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1017.648203] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4637926-9e7b-4f18-962f-cb9fac673a02 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1017.682196] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ed7f76b-110a-4764-a0a4-c762e8d86721 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1017.692213] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb7d16ac-e016-448d-8431-1eeb9df8356d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1017.709408] env[64020]: DEBUG nova.compute.provider_tree [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1017.710144] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 8cc4c09ea6a84f559b34f578a70023ab in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1017.717152] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8cc4c09ea6a84f559b34f578a70023ab [ 1018.109518] env[64020]: DEBUG oslo_concurrency.lockutils [None req-1101ed3c-6c08-402b-b633-9b13ec7fcdf5 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "d585fc50-c5a6-47bc-8672-498ac81c88f1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 25.301s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1018.213720] env[64020]: DEBUG nova.scheduler.client.report [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1018.216169] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 4e83bbb215da4c3eab5fef4445d56704 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1018.230168] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e83bbb215da4c3eab5fef4445d56704 [ 1018.718856] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=64020) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1018.719083] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 4.476s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1018.719415] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 5.770s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1018.720993] env[64020]: INFO nova.compute.claims [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1018.722856] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 6f5ed75245f3497b97711965f5a97127 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1018.761736] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f5ed75245f3497b97711965f5a97127 [ 1019.226243] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 7221ebab2e6f4587beaac48f501e1c45 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1019.233807] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7221ebab2e6f4587beaac48f501e1c45 [ 1019.875007] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae6fe599-4640-4180-885c-99e96386bf55 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1019.883079] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6b7570c-9160-4ecb-87df-e40f5f9513c8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1019.918930] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-120e7c07-e59c-4cf8-8737-ff464a1044e4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1019.926560] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71d2b9d4-b4de-43d1-bb49-64218cbc0fe9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1019.940040] env[64020]: DEBUG nova.compute.provider_tree [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1019.940531] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 430d0828212f462ba90cac3b95be64bc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1019.948654] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 430d0828212f462ba90cac3b95be64bc [ 1020.301696] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "0f805838-262c-46e2-ae44-7fd602b8d446" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1020.301989] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "0f805838-262c-46e2-ae44-7fd602b8d446" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1020.302372] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 9cb3e32cd7704e65849cc0adab9fba3c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1020.309984] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9cb3e32cd7704e65849cc0adab9fba3c [ 1020.442926] env[64020]: DEBUG nova.scheduler.client.report [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1020.445226] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg dbb45016649c4ccfb675dc446ed31776 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1020.456076] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dbb45016649c4ccfb675dc446ed31776 [ 1020.804847] env[64020]: DEBUG nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1020.806757] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 96bdfd52326d457a805482e083d6b0cd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1020.839682] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 96bdfd52326d457a805482e083d6b0cd [ 1020.947932] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.228s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1020.948491] env[64020]: DEBUG nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1020.950209] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 5ad1cd35f4c6481e85bab27c9651b62b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1020.951205] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.500s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1020.952742] env[64020]: INFO nova.compute.claims [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1020.954209] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 04b5c9289cdc42e2bebecb2a996cbd48 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1020.982469] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ad1cd35f4c6481e85bab27c9651b62b [ 1020.986364] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 04b5c9289cdc42e2bebecb2a996cbd48 [ 1021.326191] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1021.457254] env[64020]: DEBUG nova.compute.utils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1021.457863] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 03c7fa320c804c79a038d655236359c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1021.460024] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg b0d28c9d7e7c43fdb755825afeb3b853 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1021.460853] env[64020]: DEBUG nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1021.461030] env[64020]: DEBUG nova.network.neutron [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1021.468530] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b0d28c9d7e7c43fdb755825afeb3b853 [ 1021.469944] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03c7fa320c804c79a038d655236359c1 [ 1021.509539] env[64020]: DEBUG nova.policy [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd496b47ef9db47fa97323baf6bbd63ed', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f1fb1191ee044c4f921f5b7935f0109d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1021.763353] env[64020]: DEBUG nova.network.neutron [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Successfully created port: cf57af07-9b0f-4a03-8e0c-4988648fafc4 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1021.961736] env[64020]: DEBUG nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1021.963524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 673ca8e858e74d808703967722780250 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1022.006284] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 673ca8e858e74d808703967722780250 [ 1022.032893] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c389acf5-d3c3-4c7d-bfe7-e5e386aa4eaa {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1022.040563] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51c39da7-2c30-4e9d-b431-71108c7a6c10 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1022.076041] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b957146-b94f-42eb-9675-92c5a7cb99ed {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1022.088093] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f4c5bd0-be16-4c00-a678-a5608b78a777 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1022.104664] env[64020]: DEBUG nova.compute.provider_tree [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1022.105173] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg c9e7cb9ea87243e384b3e3ad8b943299 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1022.113022] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c9e7cb9ea87243e384b3e3ad8b943299 [ 1022.470898] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 991e9f83713442a398b442bed9ffd27a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1022.495849] env[64020]: DEBUG nova.compute.manager [req-591569db-1039-4f70-949c-93e439f643f1 req-a9902eca-86f6-40bc-9cf2-195bd5370907 service nova] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Received event network-changed-cf57af07-9b0f-4a03-8e0c-4988648fafc4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1022.496160] env[64020]: DEBUG nova.compute.manager [req-591569db-1039-4f70-949c-93e439f643f1 req-a9902eca-86f6-40bc-9cf2-195bd5370907 service nova] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Refreshing instance network info cache due to event network-changed-cf57af07-9b0f-4a03-8e0c-4988648fafc4. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1022.496402] env[64020]: DEBUG oslo_concurrency.lockutils [req-591569db-1039-4f70-949c-93e439f643f1 req-a9902eca-86f6-40bc-9cf2-195bd5370907 service nova] Acquiring lock "refresh_cache-7ebc2bed-b2a4-4649-b873-66955c30b3f3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1022.496546] env[64020]: DEBUG oslo_concurrency.lockutils [req-591569db-1039-4f70-949c-93e439f643f1 req-a9902eca-86f6-40bc-9cf2-195bd5370907 service nova] Acquired lock "refresh_cache-7ebc2bed-b2a4-4649-b873-66955c30b3f3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1022.496706] env[64020]: DEBUG nova.network.neutron [req-591569db-1039-4f70-949c-93e439f643f1 req-a9902eca-86f6-40bc-9cf2-195bd5370907 service nova] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Refreshing network info cache for port cf57af07-9b0f-4a03-8e0c-4988648fafc4 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1022.497107] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-591569db-1039-4f70-949c-93e439f643f1 req-a9902eca-86f6-40bc-9cf2-195bd5370907 service nova] Expecting reply to msg 121cb05d58044629af6e81b482fb2e11 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1022.500150] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 991e9f83713442a398b442bed9ffd27a [ 1022.503373] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 121cb05d58044629af6e81b482fb2e11 [ 1022.607881] env[64020]: DEBUG nova.scheduler.client.report [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1022.610260] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 620418bc2453463f9f1670ace6898430 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1022.620286] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 620418bc2453463f9f1670ace6898430 [ 1022.636499] env[64020]: ERROR nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port cf57af07-9b0f-4a03-8e0c-4988648fafc4, please check neutron logs for more information. [ 1022.636499] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1022.636499] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1022.636499] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1022.636499] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1022.636499] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1022.636499] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1022.636499] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1022.636499] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1022.636499] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1022.636499] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1022.636499] env[64020]: ERROR nova.compute.manager raise self.value [ 1022.636499] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1022.636499] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1022.636499] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1022.636499] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1022.637090] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1022.637090] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1022.637090] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port cf57af07-9b0f-4a03-8e0c-4988648fafc4, please check neutron logs for more information. [ 1022.637090] env[64020]: ERROR nova.compute.manager [ 1022.637090] env[64020]: Traceback (most recent call last): [ 1022.637090] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1022.637090] env[64020]: listener.cb(fileno) [ 1022.637090] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1022.637090] env[64020]: result = function(*args, **kwargs) [ 1022.637090] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1022.637090] env[64020]: return func(*args, **kwargs) [ 1022.637090] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1022.637090] env[64020]: raise e [ 1022.637090] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1022.637090] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1022.637090] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1022.637090] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1022.637090] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1022.637090] env[64020]: with excutils.save_and_reraise_exception(): [ 1022.637090] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1022.637090] env[64020]: self.force_reraise() [ 1022.637090] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1022.637090] env[64020]: raise self.value [ 1022.637090] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1022.637090] env[64020]: updated_port = self._update_port( [ 1022.637090] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1022.637090] env[64020]: _ensure_no_port_binding_failure(port) [ 1022.637090] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1022.637090] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1022.638058] env[64020]: nova.exception.PortBindingFailed: Binding failed for port cf57af07-9b0f-4a03-8e0c-4988648fafc4, please check neutron logs for more information. [ 1022.638058] env[64020]: Removing descriptor: 17 [ 1022.975781] env[64020]: DEBUG nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1023.001394] env[64020]: DEBUG nova.virt.hardware [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1023.001660] env[64020]: DEBUG nova.virt.hardware [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1023.001818] env[64020]: DEBUG nova.virt.hardware [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1023.001995] env[64020]: DEBUG nova.virt.hardware [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1023.002137] env[64020]: DEBUG nova.virt.hardware [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1023.002302] env[64020]: DEBUG nova.virt.hardware [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1023.002524] env[64020]: DEBUG nova.virt.hardware [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1023.002679] env[64020]: DEBUG nova.virt.hardware [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1023.002840] env[64020]: DEBUG nova.virt.hardware [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1023.002999] env[64020]: DEBUG nova.virt.hardware [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1023.003162] env[64020]: DEBUG nova.virt.hardware [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1023.005713] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c578d865-f0ec-42af-998a-29c554059c2b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1023.013404] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15bfbccb-6c8d-455f-99c4-b6670b0c45f3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1023.026721] env[64020]: ERROR nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port cf57af07-9b0f-4a03-8e0c-4988648fafc4, please check neutron logs for more information. [ 1023.026721] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Traceback (most recent call last): [ 1023.026721] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1023.026721] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] yield resources [ 1023.026721] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1023.026721] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] self.driver.spawn(context, instance, image_meta, [ 1023.026721] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1023.026721] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1023.026721] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1023.026721] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] vm_ref = self.build_virtual_machine(instance, [ 1023.026721] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] vif_infos = vmwarevif.get_vif_info(self._session, [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] for vif in network_info: [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] return self._sync_wrapper(fn, *args, **kwargs) [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] self.wait() [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] self[:] = self._gt.wait() [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] return self._exit_event.wait() [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1023.027351] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] current.throw(*self._exc) [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] result = function(*args, **kwargs) [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] return func(*args, **kwargs) [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] raise e [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] nwinfo = self.network_api.allocate_for_instance( [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] created_port_ids = self._update_ports_for_instance( [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] with excutils.save_and_reraise_exception(): [ 1023.027965] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1023.029051] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] self.force_reraise() [ 1023.029051] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1023.029051] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] raise self.value [ 1023.029051] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1023.029051] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] updated_port = self._update_port( [ 1023.029051] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1023.029051] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] _ensure_no_port_binding_failure(port) [ 1023.029051] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1023.029051] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] raise exception.PortBindingFailed(port_id=port['id']) [ 1023.029051] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] nova.exception.PortBindingFailed: Binding failed for port cf57af07-9b0f-4a03-8e0c-4988648fafc4, please check neutron logs for more information. [ 1023.029051] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] [ 1023.029051] env[64020]: INFO nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Terminating instance [ 1023.029614] env[64020]: DEBUG nova.network.neutron [req-591569db-1039-4f70-949c-93e439f643f1 req-a9902eca-86f6-40bc-9cf2-195bd5370907 service nova] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1023.030660] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-7ebc2bed-b2a4-4649-b873-66955c30b3f3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1023.113177] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.162s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1023.113780] env[64020]: DEBUG nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1023.115727] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 4d77c9e23e5b481391cf16f10ca6a7a9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1023.116962] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.049s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1023.125570] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 381e6d36e5f14d2497634955d5a086fa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1023.134733] env[64020]: DEBUG nova.network.neutron [req-591569db-1039-4f70-949c-93e439f643f1 req-a9902eca-86f6-40bc-9cf2-195bd5370907 service nova] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1023.135182] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-591569db-1039-4f70-949c-93e439f643f1 req-a9902eca-86f6-40bc-9cf2-195bd5370907 service nova] Expecting reply to msg 93426b3bf2ed44298e6b2fe89cf2a39e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1023.147267] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93426b3bf2ed44298e6b2fe89cf2a39e [ 1023.151267] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d77c9e23e5b481391cf16f10ca6a7a9 [ 1023.158466] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 381e6d36e5f14d2497634955d5a086fa [ 1023.629002] env[64020]: DEBUG nova.compute.utils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1023.629667] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg e83372c2f1104097a96638b44da0d6b7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1023.633869] env[64020]: DEBUG nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1023.634055] env[64020]: DEBUG nova.network.neutron [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1023.639572] env[64020]: DEBUG oslo_concurrency.lockutils [req-591569db-1039-4f70-949c-93e439f643f1 req-a9902eca-86f6-40bc-9cf2-195bd5370907 service nova] Releasing lock "refresh_cache-7ebc2bed-b2a4-4649-b873-66955c30b3f3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1023.640194] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e83372c2f1104097a96638b44da0d6b7 [ 1023.640597] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-7ebc2bed-b2a4-4649-b873-66955c30b3f3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1023.640786] env[64020]: DEBUG nova.network.neutron [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1023.641195] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg fc8c2b86f96b427686b722bfdbe27b73 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1023.648057] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc8c2b86f96b427686b722bfdbe27b73 [ 1023.674899] env[64020]: DEBUG nova.policy [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02b993f5bbd5442a8d2c06cff095e867', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e0c4cb7c0e9941f7a056fef708be822c', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1023.694935] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35fd607f-b87c-4749-ae52-c460cdd8b670 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1023.703182] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a1d6b88-d5b4-4391-8ce2-072100a187da {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1023.732374] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-265eb2cc-7247-4c63-9bc3-b760e97203c6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1023.739080] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5cbdf434-3ff4-4552-a506-a65d8a5b6fcd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1023.751787] env[64020]: DEBUG nova.compute.provider_tree [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1023.752350] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg f5cddcf740744f40aab0f6c65b044cad in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1023.758663] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5cddcf740744f40aab0f6c65b044cad [ 1023.933850] env[64020]: DEBUG nova.network.neutron [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Successfully created port: 5ce9b7ee-9978-4bc2-8d5b-072c69ed503e {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1024.134542] env[64020]: DEBUG nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1024.136284] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 35d4bb0bc5334450b75729318be1bff6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1024.166520] env[64020]: DEBUG nova.network.neutron [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1024.183014] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35d4bb0bc5334450b75729318be1bff6 [ 1024.255002] env[64020]: DEBUG nova.scheduler.client.report [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1024.257608] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 9d349e4217ad4c839fff18145cf46221 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1024.269940] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9d349e4217ad4c839fff18145cf46221 [ 1024.370841] env[64020]: DEBUG nova.network.neutron [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1024.371414] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 64dd3e805bf74d738fe912a189b789c1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1024.379616] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64dd3e805bf74d738fe912a189b789c1 [ 1024.545918] env[64020]: DEBUG nova.compute.manager [req-9c99f50d-549f-418e-bb7b-c67c7bf008b9 req-8cf9462c-5b71-45f5-a30d-647eadea1d40 service nova] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Received event network-vif-deleted-cf57af07-9b0f-4a03-8e0c-4988648fafc4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1024.641085] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg cdbc043f3a1a43f795b4d6a8a551a8ac in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1024.669126] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cdbc043f3a1a43f795b4d6a8a551a8ac [ 1024.762874] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.646s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1024.763497] env[64020]: ERROR nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 49f6eaa0-e982-4031-95c8-109122d3ba0d, please check neutron logs for more information. [ 1024.763497] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Traceback (most recent call last): [ 1024.763497] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1024.763497] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] self.driver.spawn(context, instance, image_meta, [ 1024.763497] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1024.763497] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1024.763497] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1024.763497] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] vm_ref = self.build_virtual_machine(instance, [ 1024.763497] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1024.763497] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] vif_infos = vmwarevif.get_vif_info(self._session, [ 1024.763497] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] for vif in network_info: [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] return self._sync_wrapper(fn, *args, **kwargs) [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] self.wait() [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] self[:] = self._gt.wait() [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] return self._exit_event.wait() [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] current.throw(*self._exc) [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1024.764103] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] result = function(*args, **kwargs) [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] return func(*args, **kwargs) [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] raise e [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] nwinfo = self.network_api.allocate_for_instance( [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] created_port_ids = self._update_ports_for_instance( [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] with excutils.save_and_reraise_exception(): [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] self.force_reraise() [ 1024.764692] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1024.765657] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] raise self.value [ 1024.765657] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1024.765657] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] updated_port = self._update_port( [ 1024.765657] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1024.765657] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] _ensure_no_port_binding_failure(port) [ 1024.765657] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1024.765657] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] raise exception.PortBindingFailed(port_id=port['id']) [ 1024.765657] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] nova.exception.PortBindingFailed: Binding failed for port 49f6eaa0-e982-4031-95c8-109122d3ba0d, please check neutron logs for more information. [ 1024.765657] env[64020]: ERROR nova.compute.manager [instance: ed9362fc-3388-4c89-849b-8ef93d226331] [ 1024.765657] env[64020]: DEBUG nova.compute.utils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Binding failed for port 49f6eaa0-e982-4031-95c8-109122d3ba0d, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1024.766124] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.439s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1024.766880] env[64020]: INFO nova.compute.claims [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1024.768533] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg d23b96a83ec74d479e0685c4a2f9623c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1024.773038] env[64020]: DEBUG nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Build of instance ed9362fc-3388-4c89-849b-8ef93d226331 was re-scheduled: Binding failed for port 49f6eaa0-e982-4031-95c8-109122d3ba0d, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1024.773038] env[64020]: DEBUG nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1024.773038] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "refresh_cache-ed9362fc-3388-4c89-849b-8ef93d226331" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1024.773038] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquired lock "refresh_cache-ed9362fc-3388-4c89-849b-8ef93d226331" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1024.773412] env[64020]: DEBUG nova.network.neutron [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1024.773412] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg c2c0127961e441a3b17658d5c851e8a6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1024.784726] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2c0127961e441a3b17658d5c851e8a6 [ 1024.801278] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d23b96a83ec74d479e0685c4a2f9623c [ 1024.873582] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-7ebc2bed-b2a4-4649-b873-66955c30b3f3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1024.873990] env[64020]: DEBUG nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1024.874171] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1024.874439] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9e585f7a-d5b2-4638-b270-272c4e21c2be {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1024.883471] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3ba58dd-cf0e-449e-840e-2be2d8d2f1f9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1024.904217] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7ebc2bed-b2a4-4649-b873-66955c30b3f3 could not be found. [ 1024.904435] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1024.904616] env[64020]: INFO nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1024.904854] env[64020]: DEBUG oslo.service.loopingcall [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1024.905069] env[64020]: DEBUG nova.compute.manager [-] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1024.905163] env[64020]: DEBUG nova.network.neutron [-] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1024.925369] env[64020]: DEBUG nova.network.neutron [-] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1024.925874] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1965a78d06bb4e15a43ac71cc984b5b4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1024.933948] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1965a78d06bb4e15a43ac71cc984b5b4 [ 1024.982959] env[64020]: ERROR nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 5ce9b7ee-9978-4bc2-8d5b-072c69ed503e, please check neutron logs for more information. [ 1024.982959] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1024.982959] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1024.982959] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1024.982959] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1024.982959] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1024.982959] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1024.982959] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1024.982959] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1024.982959] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1024.982959] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1024.982959] env[64020]: ERROR nova.compute.manager raise self.value [ 1024.982959] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1024.982959] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1024.982959] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1024.982959] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1024.983612] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1024.983612] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1024.983612] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 5ce9b7ee-9978-4bc2-8d5b-072c69ed503e, please check neutron logs for more information. [ 1024.983612] env[64020]: ERROR nova.compute.manager [ 1024.983784] env[64020]: Traceback (most recent call last): [ 1024.983784] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1024.983784] env[64020]: listener.cb(fileno) [ 1024.983784] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1024.983784] env[64020]: result = function(*args, **kwargs) [ 1024.983784] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1024.983784] env[64020]: return func(*args, **kwargs) [ 1024.983784] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1024.983784] env[64020]: raise e [ 1024.983784] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1024.983784] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1024.983784] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1024.983784] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1024.983784] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1024.983784] env[64020]: with excutils.save_and_reraise_exception(): [ 1024.983784] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1024.983784] env[64020]: self.force_reraise() [ 1024.983784] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1024.983784] env[64020]: raise self.value [ 1024.983784] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1024.983784] env[64020]: updated_port = self._update_port( [ 1024.983784] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1024.983784] env[64020]: _ensure_no_port_binding_failure(port) [ 1024.983784] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1024.983784] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1024.983784] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 5ce9b7ee-9978-4bc2-8d5b-072c69ed503e, please check neutron logs for more information. [ 1024.983784] env[64020]: Removing descriptor: 17 [ 1025.144734] env[64020]: DEBUG nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1025.169542] env[64020]: DEBUG nova.virt.hardware [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1025.169789] env[64020]: DEBUG nova.virt.hardware [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1025.169943] env[64020]: DEBUG nova.virt.hardware [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1025.170122] env[64020]: DEBUG nova.virt.hardware [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1025.170267] env[64020]: DEBUG nova.virt.hardware [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1025.170414] env[64020]: DEBUG nova.virt.hardware [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1025.170613] env[64020]: DEBUG nova.virt.hardware [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1025.170769] env[64020]: DEBUG nova.virt.hardware [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1025.170929] env[64020]: DEBUG nova.virt.hardware [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1025.171085] env[64020]: DEBUG nova.virt.hardware [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1025.171262] env[64020]: DEBUG nova.virt.hardware [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1025.172117] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92f42f43-2df4-44a1-a503-3459e91fb3a5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1025.179738] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-889c659a-f16e-4d39-986f-1b03b6cd07c2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1025.192714] env[64020]: ERROR nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 5ce9b7ee-9978-4bc2-8d5b-072c69ed503e, please check neutron logs for more information. [ 1025.192714] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Traceback (most recent call last): [ 1025.192714] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1025.192714] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] yield resources [ 1025.192714] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1025.192714] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] self.driver.spawn(context, instance, image_meta, [ 1025.192714] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1025.192714] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1025.192714] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1025.192714] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] vm_ref = self.build_virtual_machine(instance, [ 1025.192714] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] vif_infos = vmwarevif.get_vif_info(self._session, [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] for vif in network_info: [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] return self._sync_wrapper(fn, *args, **kwargs) [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] self.wait() [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] self[:] = self._gt.wait() [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] return self._exit_event.wait() [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1025.193336] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] current.throw(*self._exc) [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] result = function(*args, **kwargs) [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] return func(*args, **kwargs) [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] raise e [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] nwinfo = self.network_api.allocate_for_instance( [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] created_port_ids = self._update_ports_for_instance( [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] with excutils.save_and_reraise_exception(): [ 1025.193947] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1025.194547] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] self.force_reraise() [ 1025.194547] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1025.194547] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] raise self.value [ 1025.194547] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1025.194547] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] updated_port = self._update_port( [ 1025.194547] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1025.194547] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] _ensure_no_port_binding_failure(port) [ 1025.194547] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1025.194547] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] raise exception.PortBindingFailed(port_id=port['id']) [ 1025.194547] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] nova.exception.PortBindingFailed: Binding failed for port 5ce9b7ee-9978-4bc2-8d5b-072c69ed503e, please check neutron logs for more information. [ 1025.194547] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] [ 1025.194547] env[64020]: INFO nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Terminating instance [ 1025.195168] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-11849b75-4478-4bb5-b8eb-bb03b5c4dd57" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1025.195168] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-11849b75-4478-4bb5-b8eb-bb03b5c4dd57" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1025.195168] env[64020]: DEBUG nova.network.neutron [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1025.195547] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 9fb19c320a884c8db9027f3a87c87572 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1025.202323] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9fb19c320a884c8db9027f3a87c87572 [ 1025.275104] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 293edb352f4a478bb745af2a94ba5987 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1025.282663] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 293edb352f4a478bb745af2a94ba5987 [ 1025.299636] env[64020]: DEBUG nova.network.neutron [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1025.375938] env[64020]: DEBUG nova.network.neutron [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1025.376509] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 93c087462bfa42fd8c9e26e3d8a0818a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1025.383781] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93c087462bfa42fd8c9e26e3d8a0818a [ 1025.428398] env[64020]: DEBUG nova.network.neutron [-] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1025.428964] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8fc72b0a55054f27ab59762e891ce44b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1025.436869] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8fc72b0a55054f27ab59762e891ce44b [ 1025.711052] env[64020]: DEBUG nova.network.neutron [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1025.775464] env[64020]: DEBUG nova.network.neutron [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1025.775965] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg a4f07a212d2e413b98b9ddca5d4a6092 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1025.784272] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4f07a212d2e413b98b9ddca5d4a6092 [ 1025.836035] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51880cb3-83c4-47ec-9811-75ea980fb72d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1025.842345] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a22b012-fc3b-43b9-b5dc-a114d51ddd3e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1025.872376] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41bba0fa-ce1f-44f0-af31-cf4fe79147dd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1025.878368] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Releasing lock "refresh_cache-ed9362fc-3388-4c89-849b-8ef93d226331" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1025.878581] env[64020]: DEBUG nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1025.878761] env[64020]: DEBUG nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1025.878920] env[64020]: DEBUG nova.network.neutron [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1025.881416] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d259f43-9a1e-40c7-86eb-4a9029c9889f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1025.894128] env[64020]: DEBUG nova.compute.provider_tree [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1025.894578] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 9bd4f9e9576d4877b9295871d7ce5778 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1025.897921] env[64020]: DEBUG nova.network.neutron [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1025.898392] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 21b988b79fc044398dacd85ee1248907 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1025.901766] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9bd4f9e9576d4877b9295871d7ce5778 [ 1025.906240] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21b988b79fc044398dacd85ee1248907 [ 1025.930924] env[64020]: INFO nova.compute.manager [-] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Took 1.03 seconds to deallocate network for instance. [ 1025.932855] env[64020]: DEBUG nova.compute.claims [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1025.933035] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1026.279022] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-11849b75-4478-4bb5-b8eb-bb03b5c4dd57" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1026.279382] env[64020]: DEBUG nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1026.279576] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1026.279886] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4565be27-697c-49ea-afea-083a133bfa13 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1026.289578] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-730f97d7-4cef-49b4-9d38-5e4dfdb071f9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1026.309571] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 11849b75-4478-4bb5-b8eb-bb03b5c4dd57 could not be found. [ 1026.309771] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1026.309948] env[64020]: INFO nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1026.310197] env[64020]: DEBUG oslo.service.loopingcall [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1026.310418] env[64020]: DEBUG nova.compute.manager [-] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1026.310499] env[64020]: DEBUG nova.network.neutron [-] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1026.325834] env[64020]: DEBUG nova.network.neutron [-] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1026.326300] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 9be0050d57d746629ba5c8517e4b2541 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1026.333097] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9be0050d57d746629ba5c8517e4b2541 [ 1026.396893] env[64020]: DEBUG nova.scheduler.client.report [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1026.399458] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 3cd3cc7d69754092a5b8358e02bcfe89 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1026.401957] env[64020]: DEBUG nova.network.neutron [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1026.402426] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 1603f79a2d534f3aa91563e221c7ea53 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1026.411418] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1603f79a2d534f3aa91563e221c7ea53 [ 1026.411905] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3cd3cc7d69754092a5b8358e02bcfe89 [ 1026.571041] env[64020]: DEBUG nova.compute.manager [req-867a4889-e177-4d86-afa0-a4048179b601 req-0563d2f3-6c71-4135-a946-e7010595636f service nova] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Received event network-changed-5ce9b7ee-9978-4bc2-8d5b-072c69ed503e {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1026.571270] env[64020]: DEBUG nova.compute.manager [req-867a4889-e177-4d86-afa0-a4048179b601 req-0563d2f3-6c71-4135-a946-e7010595636f service nova] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Refreshing instance network info cache due to event network-changed-5ce9b7ee-9978-4bc2-8d5b-072c69ed503e. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1026.571474] env[64020]: DEBUG oslo_concurrency.lockutils [req-867a4889-e177-4d86-afa0-a4048179b601 req-0563d2f3-6c71-4135-a946-e7010595636f service nova] Acquiring lock "refresh_cache-11849b75-4478-4bb5-b8eb-bb03b5c4dd57" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1026.571618] env[64020]: DEBUG oslo_concurrency.lockutils [req-867a4889-e177-4d86-afa0-a4048179b601 req-0563d2f3-6c71-4135-a946-e7010595636f service nova] Acquired lock "refresh_cache-11849b75-4478-4bb5-b8eb-bb03b5c4dd57" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1026.571773] env[64020]: DEBUG nova.network.neutron [req-867a4889-e177-4d86-afa0-a4048179b601 req-0563d2f3-6c71-4135-a946-e7010595636f service nova] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Refreshing network info cache for port 5ce9b7ee-9978-4bc2-8d5b-072c69ed503e {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1026.572198] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-867a4889-e177-4d86-afa0-a4048179b601 req-0563d2f3-6c71-4135-a946-e7010595636f service nova] Expecting reply to msg e2b3af5f49094043b31d288bb443c20a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1026.578319] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e2b3af5f49094043b31d288bb443c20a [ 1026.828909] env[64020]: DEBUG nova.network.neutron [-] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1026.829362] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 74503204d94647fe927c609a19e15008 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1026.838514] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74503204d94647fe927c609a19e15008 [ 1026.901581] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.136s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1026.902065] env[64020]: DEBUG nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1026.903868] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 2678cc363ed942069622c54102d530a2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1026.904783] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.972s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1026.906438] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg d1df8e9547e04ecdaccb35d2d8a42e71 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1026.907391] env[64020]: INFO nova.compute.manager [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed9362fc-3388-4c89-849b-8ef93d226331] Took 1.03 seconds to deallocate network for instance. [ 1026.908885] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg b137d24134b24881932ad41a532959f7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1026.942646] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1df8e9547e04ecdaccb35d2d8a42e71 [ 1026.950157] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2678cc363ed942069622c54102d530a2 [ 1026.953347] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b137d24134b24881932ad41a532959f7 [ 1027.089636] env[64020]: DEBUG nova.network.neutron [req-867a4889-e177-4d86-afa0-a4048179b601 req-0563d2f3-6c71-4135-a946-e7010595636f service nova] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1027.166899] env[64020]: DEBUG nova.network.neutron [req-867a4889-e177-4d86-afa0-a4048179b601 req-0563d2f3-6c71-4135-a946-e7010595636f service nova] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1027.167419] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-867a4889-e177-4d86-afa0-a4048179b601 req-0563d2f3-6c71-4135-a946-e7010595636f service nova] Expecting reply to msg 2239dd750f05445196de315923b09bbb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1027.175685] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2239dd750f05445196de315923b09bbb [ 1027.331891] env[64020]: INFO nova.compute.manager [-] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Took 1.02 seconds to deallocate network for instance. [ 1027.334195] env[64020]: DEBUG nova.compute.claims [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1027.334392] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1027.412103] env[64020]: DEBUG nova.compute.utils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1027.412829] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg f2b99a4a52c84a25ad5c41e3f4d1ee85 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1027.418099] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg b93f6fc223604a578c4648df2a9b8dfe in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1027.419364] env[64020]: DEBUG nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1027.419534] env[64020]: DEBUG nova.network.neutron [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1027.423291] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f2b99a4a52c84a25ad5c41e3f4d1ee85 [ 1027.448128] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b93f6fc223604a578c4648df2a9b8dfe [ 1027.459258] env[64020]: DEBUG nova.policy [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e9a022c35a624ebab962014dc0c72037', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8a8ad74a9f874a21838d224fa7638717', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1027.479898] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-003bcdc5-6f8c-48c3-90c2-2839696d1e95 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1027.489105] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe987fbd-e87e-4324-8c47-d1c0c1d52e5a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1027.495722] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3cc902e6b63c4bd3b8a3bfcd9c4819d8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1027.521940] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3cc902e6b63c4bd3b8a3bfcd9c4819d8 [ 1027.525165] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52dcaac1-5e7c-417e-9b89-4f009c52621c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1027.533577] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2632f4a1-bcbe-46d3-ad64-93517101f7b8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1027.546713] env[64020]: DEBUG nova.compute.provider_tree [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1027.547262] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 07cf13b29c284c89a372300ec27c7ca3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1027.553836] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 07cf13b29c284c89a372300ec27c7ca3 [ 1027.669875] env[64020]: DEBUG oslo_concurrency.lockutils [req-867a4889-e177-4d86-afa0-a4048179b601 req-0563d2f3-6c71-4135-a946-e7010595636f service nova] Releasing lock "refresh_cache-11849b75-4478-4bb5-b8eb-bb03b5c4dd57" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1027.670112] env[64020]: DEBUG nova.compute.manager [req-867a4889-e177-4d86-afa0-a4048179b601 req-0563d2f3-6c71-4135-a946-e7010595636f service nova] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Received event network-vif-deleted-5ce9b7ee-9978-4bc2-8d5b-072c69ed503e {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1027.754297] env[64020]: DEBUG nova.network.neutron [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Successfully created port: 49b06d00-5c08-468b-8014-96d77290c3b0 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1027.919837] env[64020]: DEBUG nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1027.921780] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 3f5d12fef65d4b3a86330b292327aad5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1027.940841] env[64020]: INFO nova.scheduler.client.report [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Deleted allocations for instance ed9362fc-3388-4c89-849b-8ef93d226331 [ 1027.946543] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg bb4bcd247c7545e09b1e58c1c6e90cd7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1027.954388] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f5d12fef65d4b3a86330b292327aad5 [ 1027.959643] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb4bcd247c7545e09b1e58c1c6e90cd7 [ 1028.050337] env[64020]: DEBUG nova.scheduler.client.report [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1028.054113] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 46a734384106438ea119b77d8f725599 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1028.065575] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46a734384106438ea119b77d8f725599 [ 1028.428326] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 339cb9a4964d42b1ac94003067afbef2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1028.448336] env[64020]: DEBUG oslo_concurrency.lockutils [None req-ab130542-2583-404a-a5f2-c2a647941c8a tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "ed9362fc-3388-4c89-849b-8ef93d226331" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 23.844s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1028.470208] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 339cb9a4964d42b1ac94003067afbef2 [ 1028.557417] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.652s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1028.558616] env[64020]: ERROR nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port cf57af07-9b0f-4a03-8e0c-4988648fafc4, please check neutron logs for more information. [ 1028.558616] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Traceback (most recent call last): [ 1028.558616] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1028.558616] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] self.driver.spawn(context, instance, image_meta, [ 1028.558616] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1028.558616] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1028.558616] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1028.558616] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] vm_ref = self.build_virtual_machine(instance, [ 1028.558616] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1028.558616] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] vif_infos = vmwarevif.get_vif_info(self._session, [ 1028.558616] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] for vif in network_info: [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] return self._sync_wrapper(fn, *args, **kwargs) [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] self.wait() [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] self[:] = self._gt.wait() [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] return self._exit_event.wait() [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] current.throw(*self._exc) [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1028.559020] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] result = function(*args, **kwargs) [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] return func(*args, **kwargs) [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] raise e [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] nwinfo = self.network_api.allocate_for_instance( [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] created_port_ids = self._update_ports_for_instance( [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] with excutils.save_and_reraise_exception(): [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] self.force_reraise() [ 1028.559408] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1028.559777] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] raise self.value [ 1028.559777] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1028.559777] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] updated_port = self._update_port( [ 1028.559777] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1028.559777] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] _ensure_no_port_binding_failure(port) [ 1028.559777] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1028.559777] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] raise exception.PortBindingFailed(port_id=port['id']) [ 1028.559777] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] nova.exception.PortBindingFailed: Binding failed for port cf57af07-9b0f-4a03-8e0c-4988648fafc4, please check neutron logs for more information. [ 1028.559777] env[64020]: ERROR nova.compute.manager [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] [ 1028.559777] env[64020]: DEBUG nova.compute.utils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Binding failed for port cf57af07-9b0f-4a03-8e0c-4988648fafc4, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1028.560833] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 1.226s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1028.563142] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 507805148c9340e6a88aee4c800382a9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1028.564724] env[64020]: DEBUG nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Build of instance 7ebc2bed-b2a4-4649-b873-66955c30b3f3 was re-scheduled: Binding failed for port cf57af07-9b0f-4a03-8e0c-4988648fafc4, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1028.565174] env[64020]: DEBUG nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1028.565438] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-7ebc2bed-b2a4-4649-b873-66955c30b3f3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1028.565623] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-7ebc2bed-b2a4-4649-b873-66955c30b3f3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1028.565839] env[64020]: DEBUG nova.network.neutron [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1028.566243] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 2931a06e02404a36acce2d7d7a20be27 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1028.586438] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2931a06e02404a36acce2d7d7a20be27 [ 1028.599773] env[64020]: DEBUG nova.compute.manager [req-2576d315-ba40-4d49-8d01-ea038d0a5553 req-dced7971-bc3d-49f4-96c2-b9284772db79 service nova] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Received event network-changed-49b06d00-5c08-468b-8014-96d77290c3b0 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1028.600058] env[64020]: DEBUG nova.compute.manager [req-2576d315-ba40-4d49-8d01-ea038d0a5553 req-dced7971-bc3d-49f4-96c2-b9284772db79 service nova] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Refreshing instance network info cache due to event network-changed-49b06d00-5c08-468b-8014-96d77290c3b0. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1028.600500] env[64020]: DEBUG oslo_concurrency.lockutils [req-2576d315-ba40-4d49-8d01-ea038d0a5553 req-dced7971-bc3d-49f4-96c2-b9284772db79 service nova] Acquiring lock "refresh_cache-0f805838-262c-46e2-ae44-7fd602b8d446" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1028.600847] env[64020]: DEBUG oslo_concurrency.lockutils [req-2576d315-ba40-4d49-8d01-ea038d0a5553 req-dced7971-bc3d-49f4-96c2-b9284772db79 service nova] Acquired lock "refresh_cache-0f805838-262c-46e2-ae44-7fd602b8d446" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1028.601166] env[64020]: DEBUG nova.network.neutron [req-2576d315-ba40-4d49-8d01-ea038d0a5553 req-dced7971-bc3d-49f4-96c2-b9284772db79 service nova] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Refreshing network info cache for port 49b06d00-5c08-468b-8014-96d77290c3b0 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1028.601959] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-2576d315-ba40-4d49-8d01-ea038d0a5553 req-dced7971-bc3d-49f4-96c2-b9284772db79 service nova] Expecting reply to msg d2af8f2af2444578acc81af09a241ed6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1028.614454] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 507805148c9340e6a88aee4c800382a9 [ 1028.615775] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d2af8f2af2444578acc81af09a241ed6 [ 1028.622331] env[64020]: ERROR nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 49b06d00-5c08-468b-8014-96d77290c3b0, please check neutron logs for more information. [ 1028.622331] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1028.622331] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1028.622331] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1028.622331] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1028.622331] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1028.622331] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1028.622331] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1028.622331] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1028.622331] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1028.622331] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1028.622331] env[64020]: ERROR nova.compute.manager raise self.value [ 1028.622331] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1028.622331] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1028.622331] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1028.622331] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1028.622843] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1028.622843] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1028.622843] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 49b06d00-5c08-468b-8014-96d77290c3b0, please check neutron logs for more information. [ 1028.622843] env[64020]: ERROR nova.compute.manager [ 1028.622843] env[64020]: Traceback (most recent call last): [ 1028.622843] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1028.622843] env[64020]: listener.cb(fileno) [ 1028.622843] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1028.622843] env[64020]: result = function(*args, **kwargs) [ 1028.622843] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1028.622843] env[64020]: return func(*args, **kwargs) [ 1028.622843] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1028.622843] env[64020]: raise e [ 1028.622843] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1028.622843] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1028.622843] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1028.622843] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1028.622843] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1028.622843] env[64020]: with excutils.save_and_reraise_exception(): [ 1028.622843] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1028.622843] env[64020]: self.force_reraise() [ 1028.622843] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1028.622843] env[64020]: raise self.value [ 1028.622843] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1028.622843] env[64020]: updated_port = self._update_port( [ 1028.622843] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1028.622843] env[64020]: _ensure_no_port_binding_failure(port) [ 1028.622843] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1028.622843] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1028.623722] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 49b06d00-5c08-468b-8014-96d77290c3b0, please check neutron logs for more information. [ 1028.623722] env[64020]: Removing descriptor: 17 [ 1028.932142] env[64020]: DEBUG nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1028.952381] env[64020]: DEBUG nova.virt.hardware [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1028.952713] env[64020]: DEBUG nova.virt.hardware [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1028.952932] env[64020]: DEBUG nova.virt.hardware [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1028.953178] env[64020]: DEBUG nova.virt.hardware [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1028.953406] env[64020]: DEBUG nova.virt.hardware [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1028.953979] env[64020]: DEBUG nova.virt.hardware [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1028.953979] env[64020]: DEBUG nova.virt.hardware [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1028.953979] env[64020]: DEBUG nova.virt.hardware [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1028.954438] env[64020]: DEBUG nova.virt.hardware [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1028.954438] env[64020]: DEBUG nova.virt.hardware [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1028.954670] env[64020]: DEBUG nova.virt.hardware [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1028.955556] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64c26839-5210-44f9-8c59-14b89b3fbe2e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1028.963383] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e27babb-9cc2-4cd0-8ed1-d12c171dfcfe {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1028.977595] env[64020]: ERROR nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 49b06d00-5c08-468b-8014-96d77290c3b0, please check neutron logs for more information. [ 1028.977595] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Traceback (most recent call last): [ 1028.977595] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1028.977595] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] yield resources [ 1028.977595] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1028.977595] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] self.driver.spawn(context, instance, image_meta, [ 1028.977595] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1028.977595] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1028.977595] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1028.977595] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] vm_ref = self.build_virtual_machine(instance, [ 1028.977595] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] vif_infos = vmwarevif.get_vif_info(self._session, [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] for vif in network_info: [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] return self._sync_wrapper(fn, *args, **kwargs) [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] self.wait() [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] self[:] = self._gt.wait() [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] return self._exit_event.wait() [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1028.978043] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] current.throw(*self._exc) [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] result = function(*args, **kwargs) [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] return func(*args, **kwargs) [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] raise e [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] nwinfo = self.network_api.allocate_for_instance( [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] created_port_ids = self._update_ports_for_instance( [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] with excutils.save_and_reraise_exception(): [ 1028.978809] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1028.979316] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] self.force_reraise() [ 1028.979316] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1028.979316] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] raise self.value [ 1028.979316] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1028.979316] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] updated_port = self._update_port( [ 1028.979316] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1028.979316] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] _ensure_no_port_binding_failure(port) [ 1028.979316] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1028.979316] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] raise exception.PortBindingFailed(port_id=port['id']) [ 1028.979316] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] nova.exception.PortBindingFailed: Binding failed for port 49b06d00-5c08-468b-8014-96d77290c3b0, please check neutron logs for more information. [ 1028.979316] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] [ 1028.979316] env[64020]: INFO nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Terminating instance [ 1028.979900] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "refresh_cache-0f805838-262c-46e2-ae44-7fd602b8d446" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1029.091859] env[64020]: DEBUG nova.network.neutron [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1029.119717] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dab2d4f3-0aee-4cd1-b513-f91c3fa64230 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.122989] env[64020]: DEBUG nova.network.neutron [req-2576d315-ba40-4d49-8d01-ea038d0a5553 req-dced7971-bc3d-49f4-96c2-b9284772db79 service nova] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1029.130504] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d503105-5e27-47d2-a3bb-16b459ea9572 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.166776] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e32c2df-5dad-4573-bd92-d333c17b653a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.176297] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7cfaf70b-38c6-4ce9-934a-52eb383c7f88 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.192389] env[64020]: DEBUG nova.compute.provider_tree [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1029.192981] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 60ef97dfbbe3459c8a6a2e15dbdc9654 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1029.201357] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60ef97dfbbe3459c8a6a2e15dbdc9654 [ 1029.235526] env[64020]: DEBUG nova.network.neutron [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1029.236050] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 17f517fac2c445538f967b40201fa170 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1029.245292] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 17f517fac2c445538f967b40201fa170 [ 1029.259602] env[64020]: DEBUG nova.network.neutron [req-2576d315-ba40-4d49-8d01-ea038d0a5553 req-dced7971-bc3d-49f4-96c2-b9284772db79 service nova] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1029.260105] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-2576d315-ba40-4d49-8d01-ea038d0a5553 req-dced7971-bc3d-49f4-96c2-b9284772db79 service nova] Expecting reply to msg bb3c62c4b66e49918b2f44f61e093217 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1029.267752] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb3c62c4b66e49918b2f44f61e093217 [ 1029.696374] env[64020]: DEBUG nova.scheduler.client.report [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1029.699623] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 21f2e302a1ba4c5a95f07fbfe4a662c9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1029.715333] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21f2e302a1ba4c5a95f07fbfe4a662c9 [ 1029.728779] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "ed2eb4a8-f034-4cf2-925f-2d375475a36e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1029.728986] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "ed2eb4a8-f034-4cf2-925f-2d375475a36e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1029.729472] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg c84a40e4780a4156a71f0d1f9564e8fd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1029.738479] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-7ebc2bed-b2a4-4649-b873-66955c30b3f3" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1029.738715] env[64020]: DEBUG nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1029.738911] env[64020]: DEBUG nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1029.739091] env[64020]: DEBUG nova.network.neutron [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1029.741393] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c84a40e4780a4156a71f0d1f9564e8fd [ 1029.755424] env[64020]: DEBUG nova.network.neutron [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1029.755997] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg b846cf4c68eb49fa955ecc0357b931d7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1029.762177] env[64020]: DEBUG oslo_concurrency.lockutils [req-2576d315-ba40-4d49-8d01-ea038d0a5553 req-dced7971-bc3d-49f4-96c2-b9284772db79 service nova] Releasing lock "refresh_cache-0f805838-262c-46e2-ae44-7fd602b8d446" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1029.762707] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquired lock "refresh_cache-0f805838-262c-46e2-ae44-7fd602b8d446" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1029.762813] env[64020]: DEBUG nova.network.neutron [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1029.763227] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg f0b0abd3f233444eaeb0aa2c504a1be9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1029.765631] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b846cf4c68eb49fa955ecc0357b931d7 [ 1029.773116] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f0b0abd3f233444eaeb0aa2c504a1be9 [ 1030.193305] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1030.193580] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1030.194147] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg a7a641cbd31f4d359cba31e1d5abde30 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1030.202146] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.641s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1030.202818] env[64020]: ERROR nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 5ce9b7ee-9978-4bc2-8d5b-072c69ed503e, please check neutron logs for more information. [ 1030.202818] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Traceback (most recent call last): [ 1030.202818] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1030.202818] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] self.driver.spawn(context, instance, image_meta, [ 1030.202818] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1030.202818] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1030.202818] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1030.202818] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] vm_ref = self.build_virtual_machine(instance, [ 1030.202818] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1030.202818] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] vif_infos = vmwarevif.get_vif_info(self._session, [ 1030.202818] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] for vif in network_info: [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] return self._sync_wrapper(fn, *args, **kwargs) [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] self.wait() [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] self[:] = self._gt.wait() [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] return self._exit_event.wait() [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] current.throw(*self._exc) [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1030.203191] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] result = function(*args, **kwargs) [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] return func(*args, **kwargs) [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] raise e [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] nwinfo = self.network_api.allocate_for_instance( [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] created_port_ids = self._update_ports_for_instance( [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] with excutils.save_and_reraise_exception(): [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] self.force_reraise() [ 1030.203543] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1030.203897] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] raise self.value [ 1030.203897] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1030.203897] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] updated_port = self._update_port( [ 1030.203897] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1030.203897] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] _ensure_no_port_binding_failure(port) [ 1030.203897] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1030.203897] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] raise exception.PortBindingFailed(port_id=port['id']) [ 1030.203897] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] nova.exception.PortBindingFailed: Binding failed for port 5ce9b7ee-9978-4bc2-8d5b-072c69ed503e, please check neutron logs for more information. [ 1030.203897] env[64020]: ERROR nova.compute.manager [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] [ 1030.203897] env[64020]: DEBUG nova.compute.utils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Binding failed for port 5ce9b7ee-9978-4bc2-8d5b-072c69ed503e, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1030.204951] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a7a641cbd31f4d359cba31e1d5abde30 [ 1030.205555] env[64020]: DEBUG nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Build of instance 11849b75-4478-4bb5-b8eb-bb03b5c4dd57 was re-scheduled: Binding failed for port 5ce9b7ee-9978-4bc2-8d5b-072c69ed503e, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1030.206006] env[64020]: DEBUG nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1030.206248] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-11849b75-4478-4bb5-b8eb-bb03b5c4dd57" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1030.206388] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-11849b75-4478-4bb5-b8eb-bb03b5c4dd57" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1030.206552] env[64020]: DEBUG nova.network.neutron [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1030.207068] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg e319d698283f49fc905df3f823073edc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1030.213415] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e319d698283f49fc905df3f823073edc [ 1030.230977] env[64020]: DEBUG nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1030.232664] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 88b66a29ec4c4bd2abaffbf9615effff in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1030.258482] env[64020]: DEBUG nova.network.neutron [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1030.259025] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 0ede3d4a465146969c69275dd04359c7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1030.263854] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88b66a29ec4c4bd2abaffbf9615effff [ 1030.267659] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0ede3d4a465146969c69275dd04359c7 [ 1030.281233] env[64020]: DEBUG nova.network.neutron [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1030.358275] env[64020]: DEBUG nova.network.neutron [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1030.358788] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg f6e15a296fc747fc91f537c0418de7e1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1030.366204] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f6e15a296fc747fc91f537c0418de7e1 [ 1030.632473] env[64020]: DEBUG nova.compute.manager [req-95d9ed10-cbae-4af7-ba33-427e23725ceb req-d4e55b37-8558-4768-a0f8-3b7be6d4b62e service nova] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Received event network-vif-deleted-49b06d00-5c08-468b-8014-96d77290c3b0 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1030.698318] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1030.698493] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Starting heal instance info cache {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1030.698628] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Rebuilding the list of instances to heal {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1030.699148] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 298928b7c42548bcbc5fbfbd864cf604 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1030.707726] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 298928b7c42548bcbc5fbfbd864cf604 [ 1030.722657] env[64020]: DEBUG nova.network.neutron [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1030.754262] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1030.754575] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1030.755981] env[64020]: INFO nova.compute.claims [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1030.757506] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 6fcdb490bc074c2a89014f932188fb4d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1030.760454] env[64020]: INFO nova.compute.manager [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: 7ebc2bed-b2a4-4649-b873-66955c30b3f3] Took 1.02 seconds to deallocate network for instance. [ 1030.761936] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 5a49a182a0744d44a63813f28c9c7d39 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1030.787042] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6fcdb490bc074c2a89014f932188fb4d [ 1030.792553] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a49a182a0744d44a63813f28c9c7d39 [ 1030.798182] env[64020]: DEBUG nova.network.neutron [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1030.798733] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 02e40fb2641b4cb6bc7cdb0667c1e7b0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1030.804860] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02e40fb2641b4cb6bc7cdb0667c1e7b0 [ 1030.860569] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Releasing lock "refresh_cache-0f805838-262c-46e2-ae44-7fd602b8d446" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1030.860976] env[64020]: DEBUG nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1030.861163] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1030.861515] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e5c97959-9f9e-450b-aece-f29e9a88edc3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.870675] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aca4814a-ff0c-4ec8-8a31-4d15f7394065 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.891288] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0f805838-262c-46e2-ae44-7fd602b8d446 could not be found. [ 1030.891545] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1030.891723] env[64020]: INFO nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1030.891957] env[64020]: DEBUG oslo.service.loopingcall [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1030.892509] env[64020]: DEBUG nova.compute.manager [-] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1030.892606] env[64020]: DEBUG nova.network.neutron [-] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1030.906022] env[64020]: DEBUG nova.network.neutron [-] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1030.906478] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ff6893f1250c4a8eb53ed46402425010 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1030.913454] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff6893f1250c4a8eb53ed46402425010 [ 1031.201552] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Skipping network cache update for instance because it is Building. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1031.201908] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Didn't find any instances for network info cache update. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1031.201908] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1031.202061] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1031.202294] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1031.202457] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1031.202601] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1031.202748] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1031.202877] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=64020) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1031.203018] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager.update_available_resource {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1031.203389] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 614d2ba596014241b9d2fd6a0105f0cc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1031.211732] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 614d2ba596014241b9d2fd6a0105f0cc [ 1031.265286] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 090089a0dcb14f87b154e97213b4d22c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1031.268255] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 3e74483c1eba4dc09aeddf7006c0380b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1031.273049] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 090089a0dcb14f87b154e97213b4d22c [ 1031.300308] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-11849b75-4478-4bb5-b8eb-bb03b5c4dd57" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1031.300548] env[64020]: DEBUG nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1031.300734] env[64020]: DEBUG nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1031.300897] env[64020]: DEBUG nova.network.neutron [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1031.302983] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e74483c1eba4dc09aeddf7006c0380b [ 1031.315234] env[64020]: DEBUG nova.network.neutron [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1031.315789] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 8de795dd10f74cbd923da9d1ba4e3ef7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1031.321885] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8de795dd10f74cbd923da9d1ba4e3ef7 [ 1031.408201] env[64020]: DEBUG nova.network.neutron [-] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1031.408668] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg eb8c7fa1627a4d2f842f2c4c777589cb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1031.416997] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb8c7fa1627a4d2f842f2c4c777589cb [ 1031.706268] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1031.792192] env[64020]: INFO nova.scheduler.client.report [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Deleted allocations for instance 7ebc2bed-b2a4-4649-b873-66955c30b3f3 [ 1031.800136] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 1da30d1ea84c48198a282139f6fb2faf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1031.812416] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1da30d1ea84c48198a282139f6fb2faf [ 1031.818597] env[64020]: DEBUG nova.network.neutron [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1031.819063] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 13e5ca2510474356a0d719164ce96f51 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1031.828859] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3e07cd7-e28d-4f9e-ac68-e95616490c08 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.831593] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 13e5ca2510474356a0d719164ce96f51 [ 1031.836925] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54c260c7-b992-4bbb-80cd-a44c2eca46c8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.866892] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de3c88fb-8a01-4165-84f5-a8cee938ca09 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.873577] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-657508b2-0383-4109-abb2-a92e5d16373c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.886218] env[64020]: DEBUG nova.compute.provider_tree [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1031.886682] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 41edbf353a6c48a49466efb8aebc2405 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1031.892741] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 41edbf353a6c48a49466efb8aebc2405 [ 1031.910376] env[64020]: INFO nova.compute.manager [-] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Took 1.02 seconds to deallocate network for instance. [ 1031.912549] env[64020]: DEBUG nova.compute.claims [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1031.912735] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1032.301807] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cf4b720f-a4e3-4036-adeb-54b948ab748d tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "7ebc2bed-b2a4-4649-b873-66955c30b3f3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 20.373s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1032.321608] env[64020]: INFO nova.compute.manager [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: 11849b75-4478-4bb5-b8eb-bb03b5c4dd57] Took 1.02 seconds to deallocate network for instance. [ 1032.323360] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg a14da0009a4e4e6184b708f87942c40b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1032.353704] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a14da0009a4e4e6184b708f87942c40b [ 1032.388863] env[64020]: DEBUG nova.scheduler.client.report [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1032.391161] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 0362366f81b946078b75159de79e4edd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1032.402679] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0362366f81b946078b75159de79e4edd [ 1032.828120] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 4c6d076db3f1490ea405c4aa1862e4a5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1032.856781] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c6d076db3f1490ea405c4aa1862e4a5 [ 1032.893543] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.139s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1032.893965] env[64020]: DEBUG nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1032.895734] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg b8a5369ea4954666be6c0cda43b3abc3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1032.896639] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 1.190s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1032.896811] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1032.896956] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=64020) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1032.897231] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.985s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1032.899329] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg de37fb8b71ae486faa23c10301a05abd in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1032.901164] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-113c43e6-c071-4303-a47f-f2ce20632be9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1032.909873] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac2d01ed-5ed3-46db-8a21-7abb4892eb9e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1032.927461] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8a5369ea4954666be6c0cda43b3abc3 [ 1032.928516] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb23351d-db88-4b42-8c1c-a0d5da90e494 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1032.932847] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de37fb8b71ae486faa23c10301a05abd [ 1032.935759] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4c508875-3fd4-43ed-96d6-3ce629df6268 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1032.965520] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181776MB free_disk=120GB free_vcpus=48 pci_devices=None {{(pid=64020) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1032.965638] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1033.346073] env[64020]: INFO nova.scheduler.client.report [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Deleted allocations for instance 11849b75-4478-4bb5-b8eb-bb03b5c4dd57 [ 1033.352028] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 137512ec256a49d8890ec3b864b921e0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1033.366044] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 137512ec256a49d8890ec3b864b921e0 [ 1033.402638] env[64020]: DEBUG nova.compute.utils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1033.403383] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 8fbeab48230241fb8c53c342a81ae8b1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1033.404305] env[64020]: DEBUG nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1033.404485] env[64020]: DEBUG nova.network.neutron [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1033.416574] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8fbeab48230241fb8c53c342a81ae8b1 [ 1033.444420] env[64020]: DEBUG nova.policy [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fd08d772c4dd4bf5986d37b8474d30a1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '540a31f288b54221a4d0348b147e40e7', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1033.452599] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f056bea3-f374-415d-addb-5fafd794c2e1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1033.459973] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ff1cf7a-da61-40b7-b44f-26a16a4b5b41 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1033.488673] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43023b09-c9c0-48ed-bfec-4fd307c09a08 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1033.496063] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-efbf405e-3b19-4923-bf01-e7e2dc5d9237 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1033.509793] env[64020]: DEBUG nova.compute.provider_tree [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1033.510260] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 80c87d4874084b1482fbfe31fc578ef3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1033.517088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80c87d4874084b1482fbfe31fc578ef3 [ 1033.697701] env[64020]: DEBUG nova.network.neutron [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Successfully created port: 2703ce63-0adc-4631-8987-fab97a588dc9 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1033.786223] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1033.786444] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1033.786890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 70f6a111c5e74f5596159493e3d6da41 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1033.797084] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 70f6a111c5e74f5596159493e3d6da41 [ 1033.854365] env[64020]: DEBUG oslo_concurrency.lockutils [None req-64be2b6d-8ee4-4c0b-8c25-03c19a8e36d9 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "11849b75-4478-4bb5-b8eb-bb03b5c4dd57" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 20.424s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1033.909663] env[64020]: DEBUG nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1033.911412] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 4e77780ee0e14968887af635b073d2e2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1033.941248] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e77780ee0e14968887af635b073d2e2 [ 1034.012702] env[64020]: DEBUG nova.scheduler.client.report [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1034.015039] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 39e9fb019ae34d0f8caf12a530b0dda5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1034.026298] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 39e9fb019ae34d0f8caf12a530b0dda5 [ 1034.289051] env[64020]: DEBUG nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1034.296831] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg b3b4afe21a4a4440bd7af20ad29c257f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1034.329529] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3b4afe21a4a4440bd7af20ad29c257f [ 1034.415789] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg a0862ace7081413784af3b4396aff93c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1034.451664] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a0862ace7081413784af3b4396aff93c [ 1034.517455] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.620s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1034.518177] env[64020]: ERROR nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 49b06d00-5c08-468b-8014-96d77290c3b0, please check neutron logs for more information. [ 1034.518177] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Traceback (most recent call last): [ 1034.518177] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1034.518177] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] self.driver.spawn(context, instance, image_meta, [ 1034.518177] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1034.518177] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1034.518177] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1034.518177] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] vm_ref = self.build_virtual_machine(instance, [ 1034.518177] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1034.518177] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] vif_infos = vmwarevif.get_vif_info(self._session, [ 1034.518177] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] for vif in network_info: [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] return self._sync_wrapper(fn, *args, **kwargs) [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] self.wait() [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] self[:] = self._gt.wait() [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] return self._exit_event.wait() [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] current.throw(*self._exc) [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1034.518581] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] result = function(*args, **kwargs) [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] return func(*args, **kwargs) [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] raise e [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] nwinfo = self.network_api.allocate_for_instance( [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] created_port_ids = self._update_ports_for_instance( [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] with excutils.save_and_reraise_exception(): [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] self.force_reraise() [ 1034.518964] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1034.519389] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] raise self.value [ 1034.519389] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1034.519389] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] updated_port = self._update_port( [ 1034.519389] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1034.519389] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] _ensure_no_port_binding_failure(port) [ 1034.519389] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1034.519389] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] raise exception.PortBindingFailed(port_id=port['id']) [ 1034.519389] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] nova.exception.PortBindingFailed: Binding failed for port 49b06d00-5c08-468b-8014-96d77290c3b0, please check neutron logs for more information. [ 1034.519389] env[64020]: ERROR nova.compute.manager [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] [ 1034.520062] env[64020]: DEBUG nova.compute.utils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Binding failed for port 49b06d00-5c08-468b-8014-96d77290c3b0, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1034.521284] env[64020]: DEBUG nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Build of instance 0f805838-262c-46e2-ae44-7fd602b8d446 was re-scheduled: Binding failed for port 49b06d00-5c08-468b-8014-96d77290c3b0, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1034.521731] env[64020]: DEBUG nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1034.522005] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquiring lock "refresh_cache-0f805838-262c-46e2-ae44-7fd602b8d446" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1034.522273] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Acquired lock "refresh_cache-0f805838-262c-46e2-ae44-7fd602b8d446" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1034.522388] env[64020]: DEBUG nova.network.neutron [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1034.522825] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg d86fea74f5604f7ca5f7444b765080fa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1034.524248] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 1.559s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1034.525311] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 30aba3811d00490d9389b0477e5f17e6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1034.530218] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d86fea74f5604f7ca5f7444b765080fa [ 1034.540428] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30aba3811d00490d9389b0477e5f17e6 [ 1034.693827] env[64020]: DEBUG nova.compute.manager [req-c3dbbceb-6ed6-4ce5-adf0-2a610ab9944d req-e8a83baa-f270-4e77-a0b2-f87ca6f1e847 service nova] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Received event network-changed-2703ce63-0adc-4631-8987-fab97a588dc9 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1034.694110] env[64020]: DEBUG nova.compute.manager [req-c3dbbceb-6ed6-4ce5-adf0-2a610ab9944d req-e8a83baa-f270-4e77-a0b2-f87ca6f1e847 service nova] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Refreshing instance network info cache due to event network-changed-2703ce63-0adc-4631-8987-fab97a588dc9. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1034.694360] env[64020]: DEBUG oslo_concurrency.lockutils [req-c3dbbceb-6ed6-4ce5-adf0-2a610ab9944d req-e8a83baa-f270-4e77-a0b2-f87ca6f1e847 service nova] Acquiring lock "refresh_cache-ed2eb4a8-f034-4cf2-925f-2d375475a36e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1034.694566] env[64020]: DEBUG oslo_concurrency.lockutils [req-c3dbbceb-6ed6-4ce5-adf0-2a610ab9944d req-e8a83baa-f270-4e77-a0b2-f87ca6f1e847 service nova] Acquired lock "refresh_cache-ed2eb4a8-f034-4cf2-925f-2d375475a36e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1034.694823] env[64020]: DEBUG nova.network.neutron [req-c3dbbceb-6ed6-4ce5-adf0-2a610ab9944d req-e8a83baa-f270-4e77-a0b2-f87ca6f1e847 service nova] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Refreshing network info cache for port 2703ce63-0adc-4631-8987-fab97a588dc9 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1034.695308] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-c3dbbceb-6ed6-4ce5-adf0-2a610ab9944d req-e8a83baa-f270-4e77-a0b2-f87ca6f1e847 service nova] Expecting reply to msg 5aa9fb77734045799f181448906fd112 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1034.702293] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5aa9fb77734045799f181448906fd112 [ 1034.815885] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1034.863014] env[64020]: ERROR nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 2703ce63-0adc-4631-8987-fab97a588dc9, please check neutron logs for more information. [ 1034.863014] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1034.863014] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1034.863014] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1034.863014] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1034.863014] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1034.863014] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1034.863014] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1034.863014] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1034.863014] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1034.863014] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1034.863014] env[64020]: ERROR nova.compute.manager raise self.value [ 1034.863014] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1034.863014] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1034.863014] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1034.863014] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1034.863589] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1034.863589] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1034.863589] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 2703ce63-0adc-4631-8987-fab97a588dc9, please check neutron logs for more information. [ 1034.863589] env[64020]: ERROR nova.compute.manager [ 1034.863589] env[64020]: Traceback (most recent call last): [ 1034.863589] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1034.863589] env[64020]: listener.cb(fileno) [ 1034.863589] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1034.863589] env[64020]: result = function(*args, **kwargs) [ 1034.863589] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1034.863589] env[64020]: return func(*args, **kwargs) [ 1034.863589] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1034.863589] env[64020]: raise e [ 1034.863589] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1034.863589] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1034.863589] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1034.863589] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1034.863589] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1034.863589] env[64020]: with excutils.save_and_reraise_exception(): [ 1034.863589] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1034.863589] env[64020]: self.force_reraise() [ 1034.863589] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1034.863589] env[64020]: raise self.value [ 1034.863589] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1034.863589] env[64020]: updated_port = self._update_port( [ 1034.863589] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1034.863589] env[64020]: _ensure_no_port_binding_failure(port) [ 1034.863589] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1034.863589] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1034.864541] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 2703ce63-0adc-4631-8987-fab97a588dc9, please check neutron logs for more information. [ 1034.864541] env[64020]: Removing descriptor: 16 [ 1034.918713] env[64020]: DEBUG nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1034.938372] env[64020]: DEBUG nova.virt.hardware [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1034.938606] env[64020]: DEBUG nova.virt.hardware [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1034.938760] env[64020]: DEBUG nova.virt.hardware [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1034.938933] env[64020]: DEBUG nova.virt.hardware [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1034.939075] env[64020]: DEBUG nova.virt.hardware [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1034.939294] env[64020]: DEBUG nova.virt.hardware [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1034.939508] env[64020]: DEBUG nova.virt.hardware [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1034.939661] env[64020]: DEBUG nova.virt.hardware [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1034.939819] env[64020]: DEBUG nova.virt.hardware [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1034.939977] env[64020]: DEBUG nova.virt.hardware [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1034.940198] env[64020]: DEBUG nova.virt.hardware [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1034.941035] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c75e2f7e-8bc5-4e43-af3d-c762b162ccad {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1034.948757] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0509673-f420-4d29-8807-4ac06735d9d8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1034.964573] env[64020]: ERROR nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 2703ce63-0adc-4631-8987-fab97a588dc9, please check neutron logs for more information. [ 1034.964573] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Traceback (most recent call last): [ 1034.964573] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1034.964573] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] yield resources [ 1034.964573] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1034.964573] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] self.driver.spawn(context, instance, image_meta, [ 1034.964573] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1034.964573] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1034.964573] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1034.964573] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] vm_ref = self.build_virtual_machine(instance, [ 1034.964573] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] vif_infos = vmwarevif.get_vif_info(self._session, [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] for vif in network_info: [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] return self._sync_wrapper(fn, *args, **kwargs) [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] self.wait() [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] self[:] = self._gt.wait() [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] return self._exit_event.wait() [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1034.965052] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] current.throw(*self._exc) [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] result = function(*args, **kwargs) [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] return func(*args, **kwargs) [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] raise e [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] nwinfo = self.network_api.allocate_for_instance( [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] created_port_ids = self._update_ports_for_instance( [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] with excutils.save_and_reraise_exception(): [ 1034.965492] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1034.965923] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] self.force_reraise() [ 1034.965923] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1034.965923] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] raise self.value [ 1034.965923] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1034.965923] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] updated_port = self._update_port( [ 1034.965923] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1034.965923] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] _ensure_no_port_binding_failure(port) [ 1034.965923] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1034.965923] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] raise exception.PortBindingFailed(port_id=port['id']) [ 1034.965923] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] nova.exception.PortBindingFailed: Binding failed for port 2703ce63-0adc-4631-8987-fab97a588dc9, please check neutron logs for more information. [ 1034.965923] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] [ 1034.965923] env[64020]: INFO nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Terminating instance [ 1034.966747] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "refresh_cache-ed2eb4a8-f034-4cf2-925f-2d375475a36e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1035.032570] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 0c5175f71b6e4eea82dd48660bc8f825 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1035.040970] env[64020]: DEBUG nova.network.neutron [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1035.043770] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c5175f71b6e4eea82dd48660bc8f825 [ 1035.123558] env[64020]: DEBUG nova.network.neutron [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1035.124110] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg d6db5b6735e04277983c5b16e4bc34fe in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1035.132795] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d6db5b6735e04277983c5b16e4bc34fe [ 1035.213351] env[64020]: DEBUG nova.network.neutron [req-c3dbbceb-6ed6-4ce5-adf0-2a610ab9944d req-e8a83baa-f270-4e77-a0b2-f87ca6f1e847 service nova] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1035.281220] env[64020]: DEBUG nova.network.neutron [req-c3dbbceb-6ed6-4ce5-adf0-2a610ab9944d req-e8a83baa-f270-4e77-a0b2-f87ca6f1e847 service nova] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1035.281872] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-c3dbbceb-6ed6-4ce5-adf0-2a610ab9944d req-e8a83baa-f270-4e77-a0b2-f87ca6f1e847 service nova] Expecting reply to msg 87d35aae376446419ec6c05a3fe56098 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1035.290509] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 87d35aae376446419ec6c05a3fe56098 [ 1035.477663] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "a2b0c509-2ff8-4a46-b136-1e3ca4ede932" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1035.477969] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "a2b0c509-2ff8-4a46-b136-1e3ca4ede932" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1035.478329] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 3f66c06f035542249c74ba00ae8789c0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1035.486881] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f66c06f035542249c74ba00ae8789c0 [ 1035.554358] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 3e8860ed3b8c439aaf08c255f2893c2d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1035.564326] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e8860ed3b8c439aaf08c255f2893c2d [ 1035.626095] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Releasing lock "refresh_cache-0f805838-262c-46e2-ae44-7fd602b8d446" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1035.626327] env[64020]: DEBUG nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1035.626546] env[64020]: DEBUG nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1035.626713] env[64020]: DEBUG nova.network.neutron [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1035.642191] env[64020]: DEBUG nova.network.neutron [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1035.642790] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg c7673ac87e484e8db6cb2e39807ad3f7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1035.650920] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c7673ac87e484e8db6cb2e39807ad3f7 [ 1035.783973] env[64020]: DEBUG oslo_concurrency.lockutils [req-c3dbbceb-6ed6-4ce5-adf0-2a610ab9944d req-e8a83baa-f270-4e77-a0b2-f87ca6f1e847 service nova] Releasing lock "refresh_cache-ed2eb4a8-f034-4cf2-925f-2d375475a36e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1035.784627] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquired lock "refresh_cache-ed2eb4a8-f034-4cf2-925f-2d375475a36e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1035.784972] env[64020]: DEBUG nova.network.neutron [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1035.785513] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 681acd295c50482ea21b0d8a1d0c0cd2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1035.794005] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 681acd295c50482ea21b0d8a1d0c0cd2 [ 1035.980775] env[64020]: DEBUG nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1035.982734] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 7d1d0ea4a8df4809a90d8d2bec6cc124 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1036.013639] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d1d0ea4a8df4809a90d8d2bec6cc124 [ 1036.056566] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance 0f805838-262c-46e2-ae44-7fd602b8d446 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1036.056963] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance ed2eb4a8-f034-4cf2-925f-2d375475a36e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1036.057612] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 47e809f41e064814b77cecad2d1c537d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1036.067700] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47e809f41e064814b77cecad2d1c537d [ 1036.144351] env[64020]: DEBUG nova.network.neutron [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1036.145035] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg db18637054024f42b2639c6dc2d237f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1036.153734] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db18637054024f42b2639c6dc2d237f5 [ 1036.302951] env[64020]: DEBUG nova.network.neutron [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1036.370956] env[64020]: DEBUG nova.network.neutron [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1036.371704] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg e9a540d921eb43fe99107ac4e0768495 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1036.379712] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9a540d921eb43fe99107ac4e0768495 [ 1036.500689] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1036.559793] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance bcf9a116-82eb-4bf0-bbf6-6c042fec3c91 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1036.560581] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 6a53679f0fe7422f9f72332d5e6308b2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1036.570677] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a53679f0fe7422f9f72332d5e6308b2 [ 1036.647912] env[64020]: INFO nova.compute.manager [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] [instance: 0f805838-262c-46e2-ae44-7fd602b8d446] Took 1.02 seconds to deallocate network for instance. [ 1036.650482] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 5e03fbbed5004e99bfba57ca52d216a8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1036.681974] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e03fbbed5004e99bfba57ca52d216a8 [ 1036.722622] env[64020]: DEBUG nova.compute.manager [req-a974ab27-e132-435f-93ad-4c415457c245 req-526d90c8-3352-4a58-aa51-4c674ecbde29 service nova] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Received event network-vif-deleted-2703ce63-0adc-4631-8987-fab97a588dc9 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1036.873871] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Releasing lock "refresh_cache-ed2eb4a8-f034-4cf2-925f-2d375475a36e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1036.874548] env[64020]: DEBUG nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1036.874944] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1036.875376] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-697710cb-593d-4f40-bce6-a89d089a0238 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1036.886069] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-266ea1ff-f06f-492e-9202-59bb7282ae5b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1036.906216] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ed2eb4a8-f034-4cf2-925f-2d375475a36e could not be found. [ 1036.906539] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1036.906850] env[64020]: INFO nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1036.907190] env[64020]: DEBUG oslo.service.loopingcall [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1036.907518] env[64020]: DEBUG nova.compute.manager [-] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1036.907712] env[64020]: DEBUG nova.network.neutron [-] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1036.922409] env[64020]: DEBUG nova.network.neutron [-] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1036.922971] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c4481fab55bd4879bf85454583fcf4a8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1036.929520] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c4481fab55bd4879bf85454583fcf4a8 [ 1037.063625] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Instance a2b0c509-2ff8-4a46-b136-1e3ca4ede932 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=64020) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1037.064195] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Total usable vcpus: 48, total allocated vcpus: 1 {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1037.064553] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=704MB phys_disk=200GB used_disk=1GB total_vcpus=48 used_vcpus=1 pci_stats=[] {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1037.119738] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab5bc94e-1986-43df-bb2d-566f409fa522 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1037.127364] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35451ad1-0bea-4f20-a0a9-c67b73101c6f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1037.157386] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 7040b58469f6428484464dd95cb47e72 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1037.159737] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60a03f40-0238-4c8f-9764-2eb5ffa0394e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1037.166879] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7fad921-dffe-479d-89e2-e498f7af4b7a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1037.179427] env[64020]: DEBUG nova.compute.provider_tree [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1037.180044] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 12bb11d142be46d0939cb3cef49a514e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1037.187633] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 12bb11d142be46d0939cb3cef49a514e [ 1037.189898] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7040b58469f6428484464dd95cb47e72 [ 1037.425177] env[64020]: DEBUG nova.network.neutron [-] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1037.426243] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bd9748da37ad4d83944126ce3c1573ae in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1037.435915] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bd9748da37ad4d83944126ce3c1573ae [ 1037.679884] env[64020]: INFO nova.scheduler.client.report [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Deleted allocations for instance 0f805838-262c-46e2-ae44-7fd602b8d446 [ 1037.687875] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Expecting reply to msg 753c2e1351054f5ea9f709be4c30efd1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1037.689711] env[64020]: DEBUG nova.scheduler.client.report [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1037.692060] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg e3274f0990c04bd0bebf423dfc92e45e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1037.697917] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 753c2e1351054f5ea9f709be4c30efd1 [ 1037.711977] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3274f0990c04bd0bebf423dfc92e45e [ 1037.928739] env[64020]: INFO nova.compute.manager [-] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Took 1.02 seconds to deallocate network for instance. [ 1037.931620] env[64020]: DEBUG nova.compute.claims [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1037.931963] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1038.194920] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=64020) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1038.195148] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 3.671s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1038.195503] env[64020]: DEBUG oslo_concurrency.lockutils [None req-355e3528-6b58-4662-b35f-f45e299334d6 tempest-AttachInterfacesTestJSON-1566709644 tempest-AttachInterfacesTestJSON-1566709644-project-member] Lock "0f805838-262c-46e2-ae44-7fd602b8d446" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 17.894s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1038.195732] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.380s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1038.197240] env[64020]: INFO nova.compute.claims [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1038.198778] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg e3bcbd57defe468f8a7882060379e8b2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1038.239050] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3bcbd57defe468f8a7882060379e8b2 [ 1038.704447] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 11559a1a13f54905aaa61acce5bd54a9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1038.711056] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 11559a1a13f54905aaa61acce5bd54a9 [ 1039.263267] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d40402c-3774-487b-a29d-844d761bfaf6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1039.270967] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be1337aa-1bd9-40fd-b504-72fa4c714f61 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1039.304692] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84dc955a-9a49-4cd4-a6ee-1e2023c005db {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1039.311968] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bebc5524-894c-4743-9285-134b78fa1d6b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1039.326119] env[64020]: DEBUG nova.compute.provider_tree [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1039.326611] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg d36b023915194ac7bde2a16df8926cc9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1039.336099] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d36b023915194ac7bde2a16df8926cc9 [ 1039.829517] env[64020]: DEBUG nova.scheduler.client.report [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1039.832194] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg eca5c479679a46a0a69c45f847954d9a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1039.846088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eca5c479679a46a0a69c45f847954d9a [ 1040.336053] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.139s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1040.336053] env[64020]: DEBUG nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1040.337571] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 04fa8930210a4bec9d5ac5cc55973274 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1040.338586] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.838s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1040.341459] env[64020]: INFO nova.compute.claims [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1040.342934] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg ebcf5e65f4874b4eaa4f805195ebec54 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1040.371203] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 04fa8930210a4bec9d5ac5cc55973274 [ 1040.377651] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ebcf5e65f4874b4eaa4f805195ebec54 [ 1040.846136] env[64020]: DEBUG nova.compute.utils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1040.846770] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 2e318417cb81464292824e054f239975 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1040.848739] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 366a07c3151140a08ec6673ccad2a20b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1040.849773] env[64020]: DEBUG nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1040.849942] env[64020]: DEBUG nova.network.neutron [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1040.856890] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 366a07c3151140a08ec6673ccad2a20b [ 1040.858748] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2e318417cb81464292824e054f239975 [ 1040.990099] env[64020]: DEBUG nova.policy [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd496b47ef9db47fa97323baf6bbd63ed', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f1fb1191ee044c4f921f5b7935f0109d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1041.351407] env[64020]: DEBUG nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1041.353187] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 69c12a22eaf8417a823668f66142b99f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1041.418552] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69c12a22eaf8417a823668f66142b99f [ 1041.501610] env[64020]: DEBUG nova.network.neutron [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Successfully created port: 9a1bed53-004e-4198-bcb6-e0a191555beb {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1041.501875] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c8a0503-0ea7-4db6-a548-b5801b3c2a91 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.510221] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61e8b1a5-cc50-4d59-a472-421a57983d9a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.544779] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd666609-8549-488d-807e-dc21bf170d40 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.552860] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3b17040-b2f3-41a0-ba16-848dabf8eb6f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.568511] env[64020]: DEBUG nova.compute.provider_tree [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1041.569376] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 75a2705a0cb7440a9b744290dbb406f4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1041.576872] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 75a2705a0cb7440a9b744290dbb406f4 [ 1041.861196] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 8d5cee5200a648939874e6578e9dcc2c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1041.895855] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d5cee5200a648939874e6578e9dcc2c [ 1042.072506] env[64020]: DEBUG nova.scheduler.client.report [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1042.075008] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg ef79a88d343542b0bd4779a94c76eee3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1042.085882] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef79a88d343542b0bd4779a94c76eee3 [ 1042.366317] env[64020]: DEBUG nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1042.394146] env[64020]: DEBUG nova.virt.hardware [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1042.394478] env[64020]: DEBUG nova.virt.hardware [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1042.394683] env[64020]: DEBUG nova.virt.hardware [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1042.394908] env[64020]: DEBUG nova.virt.hardware [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1042.395091] env[64020]: DEBUG nova.virt.hardware [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1042.395271] env[64020]: DEBUG nova.virt.hardware [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1042.395519] env[64020]: DEBUG nova.virt.hardware [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1042.395717] env[64020]: DEBUG nova.virt.hardware [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1042.395923] env[64020]: DEBUG nova.virt.hardware [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1042.396139] env[64020]: DEBUG nova.virt.hardware [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1042.396357] env[64020]: DEBUG nova.virt.hardware [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1042.397712] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1624efd6-ef72-4cbf-b080-00e25236ce91 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.407862] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b19bebb-d6b8-44c1-a22d-9d8a30f92611 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.577557] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.239s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1042.578252] env[64020]: DEBUG nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1042.580186] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 5aa9c531e0d341959c456f16dbaed46d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1042.593403] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 4.649s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1042.596909] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg cc9a282aef37417e978f3fb2fe3afe46 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1042.618158] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5aa9c531e0d341959c456f16dbaed46d [ 1042.635418] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cc9a282aef37417e978f3fb2fe3afe46 [ 1042.825733] env[64020]: DEBUG nova.compute.manager [req-3926fcc4-9d31-46c3-b1ba-fd844dd5a839 req-693f5e5b-4fa9-4caf-97b7-b20ef88e8806 service nova] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Received event network-changed-9a1bed53-004e-4198-bcb6-e0a191555beb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1042.825942] env[64020]: DEBUG nova.compute.manager [req-3926fcc4-9d31-46c3-b1ba-fd844dd5a839 req-693f5e5b-4fa9-4caf-97b7-b20ef88e8806 service nova] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Refreshing instance network info cache due to event network-changed-9a1bed53-004e-4198-bcb6-e0a191555beb. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1042.826159] env[64020]: DEBUG oslo_concurrency.lockutils [req-3926fcc4-9d31-46c3-b1ba-fd844dd5a839 req-693f5e5b-4fa9-4caf-97b7-b20ef88e8806 service nova] Acquiring lock "refresh_cache-bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1042.826544] env[64020]: DEBUG oslo_concurrency.lockutils [req-3926fcc4-9d31-46c3-b1ba-fd844dd5a839 req-693f5e5b-4fa9-4caf-97b7-b20ef88e8806 service nova] Acquired lock "refresh_cache-bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1042.826779] env[64020]: DEBUG nova.network.neutron [req-3926fcc4-9d31-46c3-b1ba-fd844dd5a839 req-693f5e5b-4fa9-4caf-97b7-b20ef88e8806 service nova] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Refreshing network info cache for port 9a1bed53-004e-4198-bcb6-e0a191555beb {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1042.827204] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3926fcc4-9d31-46c3-b1ba-fd844dd5a839 req-693f5e5b-4fa9-4caf-97b7-b20ef88e8806 service nova] Expecting reply to msg 756dc8f7bfa84231bb2c4c5f0e2331b9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1042.836688] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 756dc8f7bfa84231bb2c4c5f0e2331b9 [ 1043.053244] env[64020]: ERROR nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9a1bed53-004e-4198-bcb6-e0a191555beb, please check neutron logs for more information. [ 1043.053244] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1043.053244] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1043.053244] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1043.053244] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1043.053244] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1043.053244] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1043.053244] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1043.053244] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1043.053244] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1043.053244] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1043.053244] env[64020]: ERROR nova.compute.manager raise self.value [ 1043.053244] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1043.053244] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1043.053244] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1043.053244] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1043.054047] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1043.054047] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1043.054047] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9a1bed53-004e-4198-bcb6-e0a191555beb, please check neutron logs for more information. [ 1043.054047] env[64020]: ERROR nova.compute.manager [ 1043.054047] env[64020]: Traceback (most recent call last): [ 1043.054047] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1043.054047] env[64020]: listener.cb(fileno) [ 1043.054047] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1043.054047] env[64020]: result = function(*args, **kwargs) [ 1043.054047] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1043.054047] env[64020]: return func(*args, **kwargs) [ 1043.054047] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1043.054047] env[64020]: raise e [ 1043.054047] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1043.054047] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1043.054047] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1043.054047] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1043.054047] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1043.054047] env[64020]: with excutils.save_and_reraise_exception(): [ 1043.054047] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1043.054047] env[64020]: self.force_reraise() [ 1043.054047] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1043.054047] env[64020]: raise self.value [ 1043.054047] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1043.054047] env[64020]: updated_port = self._update_port( [ 1043.054047] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1043.054047] env[64020]: _ensure_no_port_binding_failure(port) [ 1043.054047] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1043.054047] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1043.054886] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 9a1bed53-004e-4198-bcb6-e0a191555beb, please check neutron logs for more information. [ 1043.054886] env[64020]: Removing descriptor: 16 [ 1043.054886] env[64020]: ERROR nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9a1bed53-004e-4198-bcb6-e0a191555beb, please check neutron logs for more information. [ 1043.054886] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Traceback (most recent call last): [ 1043.054886] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1043.054886] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] yield resources [ 1043.054886] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1043.054886] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] self.driver.spawn(context, instance, image_meta, [ 1043.054886] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1043.054886] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1043.054886] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1043.054886] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] vm_ref = self.build_virtual_machine(instance, [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] vif_infos = vmwarevif.get_vif_info(self._session, [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] for vif in network_info: [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] return self._sync_wrapper(fn, *args, **kwargs) [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] self.wait() [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] self[:] = self._gt.wait() [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] return self._exit_event.wait() [ 1043.055254] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] result = hub.switch() [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] return self.greenlet.switch() [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] result = function(*args, **kwargs) [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] return func(*args, **kwargs) [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] raise e [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] nwinfo = self.network_api.allocate_for_instance( [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1043.055625] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] created_port_ids = self._update_ports_for_instance( [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] with excutils.save_and_reraise_exception(): [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] self.force_reraise() [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] raise self.value [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] updated_port = self._update_port( [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] _ensure_no_port_binding_failure(port) [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1043.056000] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] raise exception.PortBindingFailed(port_id=port['id']) [ 1043.056374] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] nova.exception.PortBindingFailed: Binding failed for port 9a1bed53-004e-4198-bcb6-e0a191555beb, please check neutron logs for more information. [ 1043.056374] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] [ 1043.056374] env[64020]: INFO nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Terminating instance [ 1043.057034] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1043.105329] env[64020]: DEBUG nova.compute.utils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1043.105909] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 6881bc4426fa40eab88cb9ed5c212fc5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1043.110798] env[64020]: DEBUG nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1043.111000] env[64020]: DEBUG nova.network.neutron [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1043.116648] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6881bc4426fa40eab88cb9ed5c212fc5 [ 1043.165527] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6144bc1d-93e6-4cf8-8e10-a997a8f8a4da {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1043.170330] env[64020]: DEBUG nova.policy [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02b993f5bbd5442a8d2c06cff095e867', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e0c4cb7c0e9941f7a056fef708be822c', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1043.174549] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86f79cd1-efcf-47cd-aa39-51055cb53354 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1043.211623] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ec62ee4-8030-49cd-af15-1055337a819a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1043.220358] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9c60bfb-c156-4c7b-af39-eacda6fd7ae8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1043.235577] env[64020]: DEBUG nova.compute.provider_tree [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1043.236208] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 46ce4ac3fd444b54a06ed78d86f50311 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1043.246683] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46ce4ac3fd444b54a06ed78d86f50311 [ 1043.347259] env[64020]: DEBUG nova.network.neutron [req-3926fcc4-9d31-46c3-b1ba-fd844dd5a839 req-693f5e5b-4fa9-4caf-97b7-b20ef88e8806 service nova] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1043.484176] env[64020]: DEBUG nova.network.neutron [req-3926fcc4-9d31-46c3-b1ba-fd844dd5a839 req-693f5e5b-4fa9-4caf-97b7-b20ef88e8806 service nova] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1043.484176] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-3926fcc4-9d31-46c3-b1ba-fd844dd5a839 req-693f5e5b-4fa9-4caf-97b7-b20ef88e8806 service nova] Expecting reply to msg 46b7effab80e42d99fb01f5511df09d3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1043.492308] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46b7effab80e42d99fb01f5511df09d3 [ 1043.495684] env[64020]: DEBUG nova.network.neutron [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Successfully created port: 1d547425-b7ab-4225-b7c0-da661307e5e1 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1043.615239] env[64020]: DEBUG nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1043.615239] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 33dcbb766cb74c14ad6c41da11d08fcc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1043.678689] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33dcbb766cb74c14ad6c41da11d08fcc [ 1043.742490] env[64020]: DEBUG nova.scheduler.client.report [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1043.742490] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 94235e9d1d5d4aafaab4bb51edc18d50 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1043.759656] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94235e9d1d5d4aafaab4bb51edc18d50 [ 1043.986505] env[64020]: DEBUG oslo_concurrency.lockutils [req-3926fcc4-9d31-46c3-b1ba-fd844dd5a839 req-693f5e5b-4fa9-4caf-97b7-b20ef88e8806 service nova] Releasing lock "refresh_cache-bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1043.986946] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1043.987123] env[64020]: DEBUG nova.network.neutron [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1043.987560] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 59fcf53e4bc74989a57223c90a19897b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1043.994626] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59fcf53e4bc74989a57223c90a19897b [ 1044.119206] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 1bf225d4eb3146cd8ad10b194f61cdc2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1044.159370] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1bf225d4eb3146cd8ad10b194f61cdc2 [ 1044.244305] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.663s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1044.244931] env[64020]: ERROR nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 2703ce63-0adc-4631-8987-fab97a588dc9, please check neutron logs for more information. [ 1044.244931] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Traceback (most recent call last): [ 1044.244931] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1044.244931] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] self.driver.spawn(context, instance, image_meta, [ 1044.244931] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1044.244931] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1044.244931] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1044.244931] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] vm_ref = self.build_virtual_machine(instance, [ 1044.244931] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1044.244931] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] vif_infos = vmwarevif.get_vif_info(self._session, [ 1044.244931] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] for vif in network_info: [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] return self._sync_wrapper(fn, *args, **kwargs) [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] self.wait() [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] self[:] = self._gt.wait() [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] return self._exit_event.wait() [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] current.throw(*self._exc) [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1044.245303] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] result = function(*args, **kwargs) [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] return func(*args, **kwargs) [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] raise e [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] nwinfo = self.network_api.allocate_for_instance( [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] created_port_ids = self._update_ports_for_instance( [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] with excutils.save_and_reraise_exception(): [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] self.force_reraise() [ 1044.245699] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1044.246095] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] raise self.value [ 1044.246095] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1044.246095] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] updated_port = self._update_port( [ 1044.246095] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1044.246095] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] _ensure_no_port_binding_failure(port) [ 1044.246095] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1044.246095] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] raise exception.PortBindingFailed(port_id=port['id']) [ 1044.246095] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] nova.exception.PortBindingFailed: Binding failed for port 2703ce63-0adc-4631-8987-fab97a588dc9, please check neutron logs for more information. [ 1044.246095] env[64020]: ERROR nova.compute.manager [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] [ 1044.246095] env[64020]: DEBUG nova.compute.utils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Binding failed for port 2703ce63-0adc-4631-8987-fab97a588dc9, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1044.247367] env[64020]: DEBUG nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Build of instance ed2eb4a8-f034-4cf2-925f-2d375475a36e was re-scheduled: Binding failed for port 2703ce63-0adc-4631-8987-fab97a588dc9, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1044.247756] env[64020]: DEBUG nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1044.248049] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "refresh_cache-ed2eb4a8-f034-4cf2-925f-2d375475a36e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1044.248140] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquired lock "refresh_cache-ed2eb4a8-f034-4cf2-925f-2d375475a36e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1044.248301] env[64020]: DEBUG nova.network.neutron [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1044.248703] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 9fb15757b28a41a2aefdd17588ad9b8c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1044.255935] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9fb15757b28a41a2aefdd17588ad9b8c [ 1044.525821] env[64020]: DEBUG nova.network.neutron [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1044.556862] env[64020]: ERROR nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1d547425-b7ab-4225-b7c0-da661307e5e1, please check neutron logs for more information. [ 1044.556862] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1044.556862] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1044.556862] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1044.556862] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1044.556862] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1044.556862] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1044.556862] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1044.556862] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1044.556862] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1044.556862] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1044.556862] env[64020]: ERROR nova.compute.manager raise self.value [ 1044.556862] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1044.556862] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1044.556862] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1044.556862] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1044.557416] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1044.557416] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1044.557416] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1d547425-b7ab-4225-b7c0-da661307e5e1, please check neutron logs for more information. [ 1044.557416] env[64020]: ERROR nova.compute.manager [ 1044.557416] env[64020]: Traceback (most recent call last): [ 1044.557416] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1044.557416] env[64020]: listener.cb(fileno) [ 1044.557416] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1044.557416] env[64020]: result = function(*args, **kwargs) [ 1044.557416] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1044.557416] env[64020]: return func(*args, **kwargs) [ 1044.557416] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1044.557416] env[64020]: raise e [ 1044.557416] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1044.557416] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1044.557416] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1044.557416] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1044.557416] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1044.557416] env[64020]: with excutils.save_and_reraise_exception(): [ 1044.557416] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1044.557416] env[64020]: self.force_reraise() [ 1044.557416] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1044.557416] env[64020]: raise self.value [ 1044.557416] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1044.557416] env[64020]: updated_port = self._update_port( [ 1044.557416] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1044.557416] env[64020]: _ensure_no_port_binding_failure(port) [ 1044.557416] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1044.557416] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1044.558493] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 1d547425-b7ab-4225-b7c0-da661307e5e1, please check neutron logs for more information. [ 1044.558493] env[64020]: Removing descriptor: 16 [ 1044.622586] env[64020]: DEBUG nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1044.625307] env[64020]: DEBUG nova.network.neutron [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1044.625757] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 47e11986a0274de4a895c2af8cacfbb8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1044.634761] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47e11986a0274de4a895c2af8cacfbb8 [ 1044.651545] env[64020]: DEBUG nova.virt.hardware [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1044.651803] env[64020]: DEBUG nova.virt.hardware [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1044.651899] env[64020]: DEBUG nova.virt.hardware [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1044.652090] env[64020]: DEBUG nova.virt.hardware [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1044.652242] env[64020]: DEBUG nova.virt.hardware [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1044.652414] env[64020]: DEBUG nova.virt.hardware [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1044.652614] env[64020]: DEBUG nova.virt.hardware [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1044.652764] env[64020]: DEBUG nova.virt.hardware [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1044.652924] env[64020]: DEBUG nova.virt.hardware [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1044.653079] env[64020]: DEBUG nova.virt.hardware [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1044.653262] env[64020]: DEBUG nova.virt.hardware [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1044.654331] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd23246b-a7a4-4cd9-b013-31440ee1b7ef {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1044.662393] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ae9bf8c-341e-46af-99f9-608ee1dbddd5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1044.675880] env[64020]: ERROR nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1d547425-b7ab-4225-b7c0-da661307e5e1, please check neutron logs for more information. [ 1044.675880] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Traceback (most recent call last): [ 1044.675880] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1044.675880] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] yield resources [ 1044.675880] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1044.675880] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] self.driver.spawn(context, instance, image_meta, [ 1044.675880] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1044.675880] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1044.675880] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1044.675880] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] vm_ref = self.build_virtual_machine(instance, [ 1044.675880] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] vif_infos = vmwarevif.get_vif_info(self._session, [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] for vif in network_info: [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] return self._sync_wrapper(fn, *args, **kwargs) [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] self.wait() [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] self[:] = self._gt.wait() [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] return self._exit_event.wait() [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1044.676315] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] current.throw(*self._exc) [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] result = function(*args, **kwargs) [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] return func(*args, **kwargs) [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] raise e [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] nwinfo = self.network_api.allocate_for_instance( [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] created_port_ids = self._update_ports_for_instance( [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] with excutils.save_and_reraise_exception(): [ 1044.676762] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1044.677217] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] self.force_reraise() [ 1044.677217] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1044.677217] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] raise self.value [ 1044.677217] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1044.677217] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] updated_port = self._update_port( [ 1044.677217] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1044.677217] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] _ensure_no_port_binding_failure(port) [ 1044.677217] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1044.677217] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] raise exception.PortBindingFailed(port_id=port['id']) [ 1044.677217] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] nova.exception.PortBindingFailed: Binding failed for port 1d547425-b7ab-4225-b7c0-da661307e5e1, please check neutron logs for more information. [ 1044.677217] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] [ 1044.677217] env[64020]: INFO nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Terminating instance [ 1044.678066] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-a2b0c509-2ff8-4a46-b136-1e3ca4ede932" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1044.678226] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-a2b0c509-2ff8-4a46-b136-1e3ca4ede932" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1044.678388] env[64020]: DEBUG nova.network.neutron [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1044.678784] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 6769089de9434678b70a3851cfc3f76e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1044.685356] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6769089de9434678b70a3851cfc3f76e [ 1044.722615] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Acquiring lock "f1785140-ebf1-4f2e-b35b-573d55a3e3a0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1044.722829] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Lock "f1785140-ebf1-4f2e-b35b-573d55a3e3a0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1044.723256] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 4ccb44b2ee414275b79e7e2731dea30c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1044.732712] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4ccb44b2ee414275b79e7e2731dea30c [ 1044.769375] env[64020]: DEBUG nova.network.neutron [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1044.838748] env[64020]: DEBUG nova.network.neutron [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1044.839296] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg b52dd66cf5e24b309e99a6ef3767717d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1044.851823] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b52dd66cf5e24b309e99a6ef3767717d [ 1044.859164] env[64020]: DEBUG nova.compute.manager [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Received event network-vif-deleted-9a1bed53-004e-4198-bcb6-e0a191555beb {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1044.859341] env[64020]: DEBUG nova.compute.manager [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Received event network-changed-1d547425-b7ab-4225-b7c0-da661307e5e1 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1044.859518] env[64020]: DEBUG nova.compute.manager [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Refreshing instance network info cache due to event network-changed-1d547425-b7ab-4225-b7c0-da661307e5e1. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1044.859644] env[64020]: DEBUG oslo_concurrency.lockutils [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] Acquiring lock "refresh_cache-a2b0c509-2ff8-4a46-b136-1e3ca4ede932" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1045.127668] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1045.128181] env[64020]: DEBUG nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1045.128387] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1045.128684] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c2e4ef55-620d-46d4-ac44-67b257fd1603 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1045.137602] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ddcfa6c-be01-4e09-9809-22fbea7f642f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1045.158486] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance bcf9a116-82eb-4bf0-bbf6-6c042fec3c91 could not be found. [ 1045.158705] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1045.158886] env[64020]: INFO nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1045.159122] env[64020]: DEBUG oslo.service.loopingcall [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1045.159881] env[64020]: DEBUG nova.compute.manager [-] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1045.160014] env[64020]: DEBUG nova.network.neutron [-] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1045.175137] env[64020]: DEBUG nova.network.neutron [-] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1045.175556] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 868490ae3eb947e6ad6f657ae4c4e7bb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1045.183576] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 868490ae3eb947e6ad6f657ae4c4e7bb [ 1045.198378] env[64020]: DEBUG nova.network.neutron [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1045.225066] env[64020]: DEBUG nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1045.227520] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg f5d86fa9af6541d6ae7ca4ca09c5ed39 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1045.262861] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5d86fa9af6541d6ae7ca4ca09c5ed39 [ 1045.300359] env[64020]: DEBUG nova.network.neutron [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1045.300935] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 1e8f922ed31a4b3e957d9e791787a1f8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1045.309044] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e8f922ed31a4b3e957d9e791787a1f8 [ 1045.346116] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Releasing lock "refresh_cache-ed2eb4a8-f034-4cf2-925f-2d375475a36e" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1045.346303] env[64020]: DEBUG nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1045.346484] env[64020]: DEBUG nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1045.346649] env[64020]: DEBUG nova.network.neutron [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1045.361553] env[64020]: DEBUG nova.network.neutron [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1045.362234] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg cea622b32e78442fa72ffa7d1f68533d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1045.368580] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cea622b32e78442fa72ffa7d1f68533d [ 1045.677805] env[64020]: DEBUG nova.network.neutron [-] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1045.678313] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 07b1d393c8c6476988af3502e94cd76b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1045.686711] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 07b1d393c8c6476988af3502e94cd76b [ 1045.745784] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1045.746048] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1045.747971] env[64020]: INFO nova.compute.claims [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1045.749742] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 2609fc04ca98430f9128013a9e81ae38 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1045.781574] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2609fc04ca98430f9128013a9e81ae38 [ 1045.803601] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-a2b0c509-2ff8-4a46-b136-1e3ca4ede932" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1045.804034] env[64020]: DEBUG nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1045.804237] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1045.804537] env[64020]: DEBUG oslo_concurrency.lockutils [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] Acquired lock "refresh_cache-a2b0c509-2ff8-4a46-b136-1e3ca4ede932" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1045.804709] env[64020]: DEBUG nova.network.neutron [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Refreshing network info cache for port 1d547425-b7ab-4225-b7c0-da661307e5e1 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1045.805122] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] Expecting reply to msg 612e71ba61e8407797f99eefdb47694f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1045.805874] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-01131588-5ae3-4461-9cc2-9082d73daa02 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1045.811708] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 612e71ba61e8407797f99eefdb47694f [ 1045.816459] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7209a8a-ebd8-4ce5-a260-9f62ca89cee4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1045.838650] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a2b0c509-2ff8-4a46-b136-1e3ca4ede932 could not be found. [ 1045.838848] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1045.839017] env[64020]: INFO nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1045.839290] env[64020]: DEBUG oslo.service.loopingcall [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1045.839744] env[64020]: DEBUG nova.compute.manager [-] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1045.839834] env[64020]: DEBUG nova.network.neutron [-] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1045.853738] env[64020]: DEBUG nova.network.neutron [-] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1045.854212] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3b9da58394df48a4a63473bea95bf6d3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1045.860772] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b9da58394df48a4a63473bea95bf6d3 [ 1045.864609] env[64020]: DEBUG nova.network.neutron [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1045.865057] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 79d67fd9e4d84c718247cfc8b6492369 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1045.872196] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 79d67fd9e4d84c718247cfc8b6492369 [ 1046.180902] env[64020]: INFO nova.compute.manager [-] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Took 1.02 seconds to deallocate network for instance. [ 1046.183386] env[64020]: DEBUG nova.compute.claims [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1046.183570] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1046.253483] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 71bd62bce24642de9ec17d38edd65683 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1046.260906] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 71bd62bce24642de9ec17d38edd65683 [ 1046.325034] env[64020]: DEBUG nova.network.neutron [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1046.356460] env[64020]: DEBUG nova.network.neutron [-] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1046.356904] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ff5d0d4974004d4ba636e768e74d4a7c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1046.364289] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff5d0d4974004d4ba636e768e74d4a7c [ 1046.367331] env[64020]: INFO nova.compute.manager [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ed2eb4a8-f034-4cf2-925f-2d375475a36e] Took 1.02 seconds to deallocate network for instance. [ 1046.369125] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 8ffcc50061cf43448b1acb307c4b7118 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1046.401669] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ffcc50061cf43448b1acb307c4b7118 [ 1046.402878] env[64020]: DEBUG nova.network.neutron [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1046.403338] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] Expecting reply to msg e6e24af89b094143807a40445abe7393 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1046.412270] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6e24af89b094143807a40445abe7393 [ 1046.809675] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c874e974-ab27-4a0b-b76a-58709fb8aa7e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1046.816934] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83d15de9-ead8-47c2-8c6a-f18b8ad66dfc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1046.845786] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f95886c-95a2-4091-a1d7-900f88d3dbcc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1046.852309] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ff16496-1b26-46af-95a9-c89d65db59bd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1046.864443] env[64020]: INFO nova.compute.manager [-] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Took 1.02 seconds to deallocate network for instance. [ 1046.864930] env[64020]: DEBUG nova.compute.provider_tree [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1046.865391] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg bdc5fe83b29b4a53b7e4b3bf8ae2699a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1046.868653] env[64020]: DEBUG nova.compute.claims [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1046.868823] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1046.873195] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 10fc0dcd17f64ea2b6166fd413322df7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1046.874232] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bdc5fe83b29b4a53b7e4b3bf8ae2699a [ 1046.901816] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10fc0dcd17f64ea2b6166fd413322df7 [ 1046.905821] env[64020]: DEBUG oslo_concurrency.lockutils [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] Releasing lock "refresh_cache-a2b0c509-2ff8-4a46-b136-1e3ca4ede932" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1046.906051] env[64020]: DEBUG nova.compute.manager [req-e7990605-0719-49ce-bcd9-1644d58c3d78 req-05aed963-e409-4c3e-bb23-15fd892726be service nova] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Received event network-vif-deleted-1d547425-b7ab-4225-b7c0-da661307e5e1 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1047.370204] env[64020]: DEBUG nova.scheduler.client.report [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1047.372704] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg ea5517f01d3b4e87bbd53b0f91c620da in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1047.394117] env[64020]: INFO nova.scheduler.client.report [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Deleted allocations for instance ed2eb4a8-f034-4cf2-925f-2d375475a36e [ 1047.400122] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 9200e5b3e32b462ea2c23fc19ea62e2c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1047.417082] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ea5517f01d3b4e87bbd53b0f91c620da [ 1047.419229] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9200e5b3e32b462ea2c23fc19ea62e2c [ 1047.875303] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.129s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1047.875838] env[64020]: DEBUG nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1047.877645] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 3dd4ceae0a4f4d49b743acd4b041f125 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1047.879310] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 1.696s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1047.881104] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg a737b5a32dae405fbd5227b3829cf04a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1047.905670] env[64020]: DEBUG oslo_concurrency.lockutils [None req-f6c8f591-3c7a-4786-9bb9-84dec0f92445 tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "ed2eb4a8-f034-4cf2-925f-2d375475a36e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 18.177s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1047.909275] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3dd4ceae0a4f4d49b743acd4b041f125 [ 1047.913435] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a737b5a32dae405fbd5227b3829cf04a [ 1048.380703] env[64020]: DEBUG nova.compute.utils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1048.381402] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 3b95eb60f9514f0bbb7c4f5e62325dff in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1048.383093] env[64020]: DEBUG nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1048.383301] env[64020]: DEBUG nova.network.neutron [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1048.403186] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b95eb60f9514f0bbb7c4f5e62325dff [ 1048.457520] env[64020]: DEBUG nova.policy [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '66450afa293d4cb68deea898fc672799', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '10919251734d46d8886316585e61b85f', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1048.460839] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5451e4c-9405-4a21-aa55-e9c29f938f1a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1048.469193] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-153580a6-e80a-4242-a523-23ce9740b91c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1048.499323] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-944d6306-69e5-47c6-b745-b2a350644f23 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1048.506471] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96d79fe8-cab0-4200-8053-fd7bdce6aa90 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1048.520162] env[64020]: DEBUG nova.compute.provider_tree [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1048.520691] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 01d8339711a9471a9a911507cab017c5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1048.528726] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 01d8339711a9471a9a911507cab017c5 [ 1048.757604] env[64020]: DEBUG nova.network.neutron [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Successfully created port: e7d771e4-a10e-4178-af1b-7325b18e280e {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1048.883718] env[64020]: DEBUG nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1048.887339] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 1b6ebe63dd0549a3be539823c9ada451 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1048.919077] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b6ebe63dd0549a3be539823c9ada451 [ 1049.023689] env[64020]: DEBUG nova.scheduler.client.report [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1049.026386] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg dc6d62cc63eb4eac89fa0db2b4791a24 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1049.038586] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc6d62cc63eb4eac89fa0db2b4791a24 [ 1049.392146] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg d9611c7e509f46eaa964803068b7d466 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1049.432616] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9611c7e509f46eaa964803068b7d466 [ 1049.529513] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.650s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1049.530211] env[64020]: ERROR nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9a1bed53-004e-4198-bcb6-e0a191555beb, please check neutron logs for more information. [ 1049.530211] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Traceback (most recent call last): [ 1049.530211] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1049.530211] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] self.driver.spawn(context, instance, image_meta, [ 1049.530211] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1049.530211] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1049.530211] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1049.530211] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] vm_ref = self.build_virtual_machine(instance, [ 1049.530211] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1049.530211] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] vif_infos = vmwarevif.get_vif_info(self._session, [ 1049.530211] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] for vif in network_info: [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] return self._sync_wrapper(fn, *args, **kwargs) [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] self.wait() [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] self[:] = self._gt.wait() [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] return self._exit_event.wait() [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] result = hub.switch() [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1049.530573] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] return self.greenlet.switch() [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] result = function(*args, **kwargs) [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] return func(*args, **kwargs) [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] raise e [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] nwinfo = self.network_api.allocate_for_instance( [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] created_port_ids = self._update_ports_for_instance( [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] with excutils.save_and_reraise_exception(): [ 1049.530999] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1049.531464] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] self.force_reraise() [ 1049.531464] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1049.531464] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] raise self.value [ 1049.531464] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1049.531464] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] updated_port = self._update_port( [ 1049.531464] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1049.531464] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] _ensure_no_port_binding_failure(port) [ 1049.531464] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1049.531464] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] raise exception.PortBindingFailed(port_id=port['id']) [ 1049.531464] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] nova.exception.PortBindingFailed: Binding failed for port 9a1bed53-004e-4198-bcb6-e0a191555beb, please check neutron logs for more information. [ 1049.531464] env[64020]: ERROR nova.compute.manager [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] [ 1049.531980] env[64020]: DEBUG nova.compute.utils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Binding failed for port 9a1bed53-004e-4198-bcb6-e0a191555beb, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1049.532648] env[64020]: DEBUG nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Build of instance bcf9a116-82eb-4bf0-bbf6-6c042fec3c91 was re-scheduled: Binding failed for port 9a1bed53-004e-4198-bcb6-e0a191555beb, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1049.533129] env[64020]: DEBUG nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1049.533367] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1049.533516] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1049.533673] env[64020]: DEBUG nova.network.neutron [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1049.534192] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 84c9b36208e64876a7bdd5ba406e8e6d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1049.535050] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 2.666s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1049.537286] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 8260fd965e9c400597928784d6abca32 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1049.542081] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 84c9b36208e64876a7bdd5ba406e8e6d [ 1049.568262] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8260fd965e9c400597928784d6abca32 [ 1049.608935] env[64020]: DEBUG nova.compute.manager [req-fc93fbed-014e-4a46-8016-e7c9c7530edc req-1c7f99d3-cfe3-494b-9a4b-ce45b3d0f6aa service nova] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Received event network-changed-e7d771e4-a10e-4178-af1b-7325b18e280e {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1049.609177] env[64020]: DEBUG nova.compute.manager [req-fc93fbed-014e-4a46-8016-e7c9c7530edc req-1c7f99d3-cfe3-494b-9a4b-ce45b3d0f6aa service nova] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Refreshing instance network info cache due to event network-changed-e7d771e4-a10e-4178-af1b-7325b18e280e. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1049.609420] env[64020]: DEBUG oslo_concurrency.lockutils [req-fc93fbed-014e-4a46-8016-e7c9c7530edc req-1c7f99d3-cfe3-494b-9a4b-ce45b3d0f6aa service nova] Acquiring lock "refresh_cache-f1785140-ebf1-4f2e-b35b-573d55a3e3a0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1049.609614] env[64020]: DEBUG oslo_concurrency.lockutils [req-fc93fbed-014e-4a46-8016-e7c9c7530edc req-1c7f99d3-cfe3-494b-9a4b-ce45b3d0f6aa service nova] Acquired lock "refresh_cache-f1785140-ebf1-4f2e-b35b-573d55a3e3a0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1049.609949] env[64020]: DEBUG nova.network.neutron [req-fc93fbed-014e-4a46-8016-e7c9c7530edc req-1c7f99d3-cfe3-494b-9a4b-ce45b3d0f6aa service nova] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Refreshing network info cache for port e7d771e4-a10e-4178-af1b-7325b18e280e {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1049.610300] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-fc93fbed-014e-4a46-8016-e7c9c7530edc req-1c7f99d3-cfe3-494b-9a4b-ce45b3d0f6aa service nova] Expecting reply to msg 23a36339ca394504b362d9c0aad59669 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1049.617503] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 23a36339ca394504b362d9c0aad59669 [ 1049.799204] env[64020]: ERROR nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e7d771e4-a10e-4178-af1b-7325b18e280e, please check neutron logs for more information. [ 1049.799204] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1049.799204] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1049.799204] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1049.799204] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1049.799204] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1049.799204] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1049.799204] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1049.799204] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1049.799204] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1049.799204] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1049.799204] env[64020]: ERROR nova.compute.manager raise self.value [ 1049.799204] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1049.799204] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1049.799204] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1049.799204] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1049.799758] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1049.799758] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1049.799758] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e7d771e4-a10e-4178-af1b-7325b18e280e, please check neutron logs for more information. [ 1049.799758] env[64020]: ERROR nova.compute.manager [ 1049.799758] env[64020]: Traceback (most recent call last): [ 1049.799758] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1049.799758] env[64020]: listener.cb(fileno) [ 1049.799758] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1049.799758] env[64020]: result = function(*args, **kwargs) [ 1049.799758] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1049.799758] env[64020]: return func(*args, **kwargs) [ 1049.799758] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1049.799758] env[64020]: raise e [ 1049.799758] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1049.799758] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1049.799758] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1049.799758] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1049.799758] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1049.799758] env[64020]: with excutils.save_and_reraise_exception(): [ 1049.799758] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1049.799758] env[64020]: self.force_reraise() [ 1049.799758] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1049.799758] env[64020]: raise self.value [ 1049.799758] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1049.799758] env[64020]: updated_port = self._update_port( [ 1049.799758] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1049.799758] env[64020]: _ensure_no_port_binding_failure(port) [ 1049.799758] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1049.799758] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1049.800846] env[64020]: nova.exception.PortBindingFailed: Binding failed for port e7d771e4-a10e-4178-af1b-7325b18e280e, please check neutron logs for more information. [ 1049.800846] env[64020]: Removing descriptor: 17 [ 1049.846082] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "ccffbcd1-09f3-4458-8caa-54980f26da14" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1049.846316] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "ccffbcd1-09f3-4458-8caa-54980f26da14" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1049.846766] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg f16a76f065f4475cb1bd1b355f4a1584 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1049.857967] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f16a76f065f4475cb1bd1b355f4a1584 [ 1049.895644] env[64020]: DEBUG nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1049.920288] env[64020]: DEBUG nova.virt.hardware [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1049.920535] env[64020]: DEBUG nova.virt.hardware [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1049.920688] env[64020]: DEBUG nova.virt.hardware [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1049.920866] env[64020]: DEBUG nova.virt.hardware [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1049.921009] env[64020]: DEBUG nova.virt.hardware [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1049.921154] env[64020]: DEBUG nova.virt.hardware [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1049.921433] env[64020]: DEBUG nova.virt.hardware [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1049.921605] env[64020]: DEBUG nova.virt.hardware [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1049.921799] env[64020]: DEBUG nova.virt.hardware [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1049.921967] env[64020]: DEBUG nova.virt.hardware [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1049.922136] env[64020]: DEBUG nova.virt.hardware [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1049.923034] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef297c89-a16f-4ca8-a051-84d89a868e77 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.931076] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afcc499d-1afe-4483-a990-891ae36c26d7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.944641] env[64020]: ERROR nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e7d771e4-a10e-4178-af1b-7325b18e280e, please check neutron logs for more information. [ 1049.944641] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Traceback (most recent call last): [ 1049.944641] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1049.944641] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] yield resources [ 1049.944641] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1049.944641] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] self.driver.spawn(context, instance, image_meta, [ 1049.944641] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1049.944641] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1049.944641] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1049.944641] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] vm_ref = self.build_virtual_machine(instance, [ 1049.944641] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] vif_infos = vmwarevif.get_vif_info(self._session, [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] for vif in network_info: [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] return self._sync_wrapper(fn, *args, **kwargs) [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] self.wait() [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] self[:] = self._gt.wait() [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] return self._exit_event.wait() [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1049.945092] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] current.throw(*self._exc) [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] result = function(*args, **kwargs) [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] return func(*args, **kwargs) [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] raise e [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] nwinfo = self.network_api.allocate_for_instance( [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] created_port_ids = self._update_ports_for_instance( [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] with excutils.save_and_reraise_exception(): [ 1049.945533] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1049.945974] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] self.force_reraise() [ 1049.945974] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1049.945974] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] raise self.value [ 1049.945974] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1049.945974] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] updated_port = self._update_port( [ 1049.945974] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1049.945974] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] _ensure_no_port_binding_failure(port) [ 1049.945974] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1049.945974] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] raise exception.PortBindingFailed(port_id=port['id']) [ 1049.945974] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] nova.exception.PortBindingFailed: Binding failed for port e7d771e4-a10e-4178-af1b-7325b18e280e, please check neutron logs for more information. [ 1049.945974] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] [ 1049.945974] env[64020]: INFO nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Terminating instance [ 1049.947175] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Acquiring lock "refresh_cache-f1785140-ebf1-4f2e-b35b-573d55a3e3a0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1050.064710] env[64020]: DEBUG nova.network.neutron [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1050.109009] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80d86f9c-4126-4a22-a64f-67290724b31e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1050.118112] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a9521d4-470c-40f9-903b-0b78d9bf9fcf {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1050.149736] env[64020]: DEBUG nova.network.neutron [req-fc93fbed-014e-4a46-8016-e7c9c7530edc req-1c7f99d3-cfe3-494b-9a4b-ce45b3d0f6aa service nova] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1050.151761] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98ab4406-ed3f-4f8e-a1e0-0d27f1a7b83a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1050.158658] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f90a656-f83a-48a7-92af-7dc1f2025a12 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1050.171000] env[64020]: DEBUG nova.compute.provider_tree [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1050.171470] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg c97f192aacf540f188201b53c85a7064 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1050.179600] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c97f192aacf540f188201b53c85a7064 [ 1050.201462] env[64020]: DEBUG nova.network.neutron [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1050.201942] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 18eb03bf45684e1db29b8b99cae76312 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1050.210202] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18eb03bf45684e1db29b8b99cae76312 [ 1050.241964] env[64020]: DEBUG nova.network.neutron [req-fc93fbed-014e-4a46-8016-e7c9c7530edc req-1c7f99d3-cfe3-494b-9a4b-ce45b3d0f6aa service nova] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1050.242468] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-fc93fbed-014e-4a46-8016-e7c9c7530edc req-1c7f99d3-cfe3-494b-9a4b-ce45b3d0f6aa service nova] Expecting reply to msg 3e1ebb8ef85746d6abd4cfb7cebba427 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1050.249446] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e1ebb8ef85746d6abd4cfb7cebba427 [ 1050.348503] env[64020]: DEBUG nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1050.350361] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg cb10895f600e4619a6aada34d09ad8dc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1050.384408] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cb10895f600e4619a6aada34d09ad8dc [ 1050.674579] env[64020]: DEBUG nova.scheduler.client.report [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1050.676886] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg a75f0eac84154d35838bf93e0089bda0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1050.689914] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a75f0eac84154d35838bf93e0089bda0 [ 1050.703542] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1050.703785] env[64020]: DEBUG nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1050.703969] env[64020]: DEBUG nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1050.704147] env[64020]: DEBUG nova.network.neutron [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1050.723211] env[64020]: DEBUG nova.network.neutron [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1050.723794] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg b9ff064141174885a6fd162c0f275648 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1050.730971] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9ff064141174885a6fd162c0f275648 [ 1050.744565] env[64020]: DEBUG oslo_concurrency.lockutils [req-fc93fbed-014e-4a46-8016-e7c9c7530edc req-1c7f99d3-cfe3-494b-9a4b-ce45b3d0f6aa service nova] Releasing lock "refresh_cache-f1785140-ebf1-4f2e-b35b-573d55a3e3a0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1050.744958] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Acquired lock "refresh_cache-f1785140-ebf1-4f2e-b35b-573d55a3e3a0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1050.745238] env[64020]: DEBUG nova.network.neutron [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1050.745688] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg d9ed9cdc22ff448e9e690335f2313606 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1050.752795] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9ed9cdc22ff448e9e690335f2313606 [ 1050.867624] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1051.179710] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.644s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1051.180360] env[64020]: ERROR nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1d547425-b7ab-4225-b7c0-da661307e5e1, please check neutron logs for more information. [ 1051.180360] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Traceback (most recent call last): [ 1051.180360] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1051.180360] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] self.driver.spawn(context, instance, image_meta, [ 1051.180360] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1051.180360] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1051.180360] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1051.180360] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] vm_ref = self.build_virtual_machine(instance, [ 1051.180360] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1051.180360] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] vif_infos = vmwarevif.get_vif_info(self._session, [ 1051.180360] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] for vif in network_info: [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] return self._sync_wrapper(fn, *args, **kwargs) [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] self.wait() [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] self[:] = self._gt.wait() [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] return self._exit_event.wait() [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] current.throw(*self._exc) [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1051.180753] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] result = function(*args, **kwargs) [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] return func(*args, **kwargs) [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] raise e [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] nwinfo = self.network_api.allocate_for_instance( [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] created_port_ids = self._update_ports_for_instance( [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] with excutils.save_and_reraise_exception(): [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] self.force_reraise() [ 1051.181126] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1051.181541] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] raise self.value [ 1051.181541] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1051.181541] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] updated_port = self._update_port( [ 1051.181541] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1051.181541] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] _ensure_no_port_binding_failure(port) [ 1051.181541] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1051.181541] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] raise exception.PortBindingFailed(port_id=port['id']) [ 1051.181541] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] nova.exception.PortBindingFailed: Binding failed for port 1d547425-b7ab-4225-b7c0-da661307e5e1, please check neutron logs for more information. [ 1051.181541] env[64020]: ERROR nova.compute.manager [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] [ 1051.181541] env[64020]: DEBUG nova.compute.utils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Binding failed for port 1d547425-b7ab-4225-b7c0-da661307e5e1, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1051.182900] env[64020]: DEBUG nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Build of instance a2b0c509-2ff8-4a46-b136-1e3ca4ede932 was re-scheduled: Binding failed for port 1d547425-b7ab-4225-b7c0-da661307e5e1, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1051.183308] env[64020]: DEBUG nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1051.183535] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquiring lock "refresh_cache-a2b0c509-2ff8-4a46-b136-1e3ca4ede932" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1051.183709] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Acquired lock "refresh_cache-a2b0c509-2ff8-4a46-b136-1e3ca4ede932" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1051.183872] env[64020]: DEBUG nova.network.neutron [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1051.184298] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 0e83017416e6420c87673fa310ccaf50 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1051.185445] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.318s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1051.186923] env[64020]: INFO nova.compute.claims [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1051.188392] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 367207d9b39b49e4a1712b6e6c8b6db3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1051.190882] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e83017416e6420c87673fa310ccaf50 [ 1051.218692] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 367207d9b39b49e4a1712b6e6c8b6db3 [ 1051.226265] env[64020]: DEBUG nova.network.neutron [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1051.227050] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 808d966b67f146678d3785259466cd1e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1051.234753] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 808d966b67f146678d3785259466cd1e [ 1051.264824] env[64020]: DEBUG nova.network.neutron [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1051.333428] env[64020]: DEBUG nova.network.neutron [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1051.334164] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg dff88e7eda4044189d7a1ab1a790e594 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1051.342453] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dff88e7eda4044189d7a1ab1a790e594 [ 1051.633312] env[64020]: DEBUG nova.compute.manager [req-9d53bbbb-a06c-4e27-98da-adb494b02c12 req-1a0e8bf8-935b-4b13-9616-36b5a25960fa service nova] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Received event network-vif-deleted-e7d771e4-a10e-4178-af1b-7325b18e280e {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1051.691907] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg a91d9b305c97453ba6ab0cd718f1ec81 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1051.699075] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a91d9b305c97453ba6ab0cd718f1ec81 [ 1051.700100] env[64020]: DEBUG nova.network.neutron [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1051.729106] env[64020]: INFO nova.compute.manager [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: bcf9a116-82eb-4bf0-bbf6-6c042fec3c91] Took 1.02 seconds to deallocate network for instance. [ 1051.730811] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 138d60e8bc214dd4a3a17848f687250f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1051.766728] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 138d60e8bc214dd4a3a17848f687250f [ 1051.770233] env[64020]: DEBUG nova.network.neutron [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1051.770733] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg aa3322e32d364841a26e867f32ff9282 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1051.778965] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa3322e32d364841a26e867f32ff9282 [ 1051.837484] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Releasing lock "refresh_cache-f1785140-ebf1-4f2e-b35b-573d55a3e3a0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1051.837896] env[64020]: DEBUG nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1051.838087] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1051.838372] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4db57fc5-e350-4a72-9749-a3f1f5e0a6c7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1051.847141] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77d1d838-9a03-4417-9d56-d602b2224807 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1051.866892] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f1785140-ebf1-4f2e-b35b-573d55a3e3a0 could not be found. [ 1051.867090] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1051.867264] env[64020]: INFO nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1051.867493] env[64020]: DEBUG oslo.service.loopingcall [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1051.867702] env[64020]: DEBUG nova.compute.manager [-] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1051.867793] env[64020]: DEBUG nova.network.neutron [-] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1051.881907] env[64020]: DEBUG nova.network.neutron [-] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1051.882371] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg e42c954285ae423a8381a6c6afe85419 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1051.887869] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e42c954285ae423a8381a6c6afe85419 [ 1052.236387] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg cf75a8c5cd7047f2b163b0593e2bd3ba in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1052.248377] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dcde56fd-10f7-43a2-ad98-29fb1d6612a4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1052.255748] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-637c350e-22a6-4c1e-9d70-1115c9c9184a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1052.283436] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Releasing lock "refresh_cache-a2b0c509-2ff8-4a46-b136-1e3ca4ede932" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1052.283658] env[64020]: DEBUG nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1052.283836] env[64020]: DEBUG nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1052.284012] env[64020]: DEBUG nova.network.neutron [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1052.286061] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cf75a8c5cd7047f2b163b0593e2bd3ba [ 1052.286890] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e403de85-d14f-497a-b9c3-af023298d7e4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1052.294125] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a733ae3-90cd-4dc1-923a-7abd11720336 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1052.308142] env[64020]: DEBUG nova.compute.provider_tree [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1052.308621] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 66a98ce2210149d08d3f60c166f038e8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1052.310212] env[64020]: DEBUG nova.network.neutron [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1052.310546] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 5f73cf98111e4dc3a5131448d318cf4c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1052.315806] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66a98ce2210149d08d3f60c166f038e8 [ 1052.316434] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5f73cf98111e4dc3a5131448d318cf4c [ 1052.385630] env[64020]: DEBUG nova.network.neutron [-] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1052.386084] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 669e55cb46e240a2bec83bb87f2fb050 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1052.393473] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 669e55cb46e240a2bec83bb87f2fb050 [ 1052.761772] env[64020]: INFO nova.scheduler.client.report [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Deleted allocations for instance bcf9a116-82eb-4bf0-bbf6-6c042fec3c91 [ 1052.768709] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 062a2a5f54e24a71afe3479b971d8f02 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1052.780544] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 062a2a5f54e24a71afe3479b971d8f02 [ 1052.813196] env[64020]: DEBUG nova.scheduler.client.report [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1052.815734] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 730c4f142bf3484a8256c5531832b46a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1052.817073] env[64020]: DEBUG nova.network.neutron [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1052.817268] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg 9da3b167a715421587fa94ab5f1c25ee in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1052.825013] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9da3b167a715421587fa94ab5f1c25ee [ 1052.826793] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 730c4f142bf3484a8256c5531832b46a [ 1052.888231] env[64020]: INFO nova.compute.manager [-] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Took 1.02 seconds to deallocate network for instance. [ 1052.890379] env[64020]: DEBUG nova.compute.claims [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1052.890556] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1053.271258] env[64020]: DEBUG oslo_concurrency.lockutils [None req-b3402c05-1626-4a52-ab3f-f6f5c050dfb2 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "bcf9a116-82eb-4bf0-bbf6-6c042fec3c91" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 19.484s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1053.319170] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.134s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1053.319723] env[64020]: DEBUG nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1053.321534] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 88d4048010214b80b0056156f0b54061 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1053.322738] env[64020]: INFO nova.compute.manager [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] [instance: a2b0c509-2ff8-4a46-b136-1e3ca4ede932] Took 1.04 seconds to deallocate network for instance. [ 1053.324272] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg a71781a0a72644b08cfd951972135224 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1053.325336] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.435s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1053.327141] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 71c6151ac5654f57befa31899ab71151 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1053.355040] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a71781a0a72644b08cfd951972135224 [ 1053.366125] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88d4048010214b80b0056156f0b54061 [ 1053.371443] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 71c6151ac5654f57befa31899ab71151 [ 1053.827162] env[64020]: DEBUG nova.compute.utils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1053.827788] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg d755fd1e64a24cccb94d1572e2b0bd86 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1053.828803] env[64020]: DEBUG nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1053.828978] env[64020]: DEBUG nova.network.neutron [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1053.832779] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg e82a230d779141869bb1201980c37155 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1053.839285] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d755fd1e64a24cccb94d1572e2b0bd86 [ 1053.864275] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e82a230d779141869bb1201980c37155 [ 1053.884240] env[64020]: DEBUG nova.policy [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fd08d772c4dd4bf5986d37b8474d30a1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '540a31f288b54221a4d0348b147e40e7', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1053.887147] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10188ca0-a1f2-4921-b17d-1b4b28996902 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1053.894956] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83928b42-cdc4-4eca-81db-b85459c3cb69 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1053.925286] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1613b1ea-eaa9-40ae-b523-c12acbeaa737 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1053.933082] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-223691c1-0d81-4b55-adca-b6b6584d0a57 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1053.945490] env[64020]: DEBUG nova.compute.provider_tree [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1053.945988] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 3200995865444e02a3f4edb493884ea0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1053.953175] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3200995865444e02a3f4edb493884ea0 [ 1054.243589] env[64020]: DEBUG nova.network.neutron [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Successfully created port: 1b3ab288-c471-42f1-867c-364ca86380aa {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1054.339975] env[64020]: DEBUG nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1054.341749] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 400ba0b98c4a4fdb8af37dd26d7a43e1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1054.363532] env[64020]: INFO nova.scheduler.client.report [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Deleted allocations for instance a2b0c509-2ff8-4a46-b136-1e3ca4ede932 [ 1054.369572] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Expecting reply to msg f3b08a3219fd44d2af240f919b033965 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1054.397306] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3b08a3219fd44d2af240f919b033965 [ 1054.400901] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 400ba0b98c4a4fdb8af37dd26d7a43e1 [ 1054.449040] env[64020]: DEBUG nova.scheduler.client.report [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1054.451408] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg d68aceb340d74bb8bc48fc32028ea9bb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1054.467604] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d68aceb340d74bb8bc48fc32028ea9bb [ 1054.848339] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg dc75896397d44e928567ab345c8f9236 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1054.881217] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc75896397d44e928567ab345c8f9236 [ 1054.881217] env[64020]: DEBUG oslo_concurrency.lockutils [None req-df75438f-5559-4633-987a-052ae9091427 tempest-ServersTestJSON-1130810769 tempest-ServersTestJSON-1130810769-project-member] Lock "a2b0c509-2ff8-4a46-b136-1e3ca4ede932" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 19.403s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1054.950018] env[64020]: DEBUG nova.compute.manager [req-4968847b-acc1-46f0-9121-7309e91a41d0 req-42f7a202-20fb-4ec8-a359-a2347600cdf5 service nova] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Received event network-changed-1b3ab288-c471-42f1-867c-364ca86380aa {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1054.950401] env[64020]: DEBUG nova.compute.manager [req-4968847b-acc1-46f0-9121-7309e91a41d0 req-42f7a202-20fb-4ec8-a359-a2347600cdf5 service nova] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Refreshing instance network info cache due to event network-changed-1b3ab288-c471-42f1-867c-364ca86380aa. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1054.950733] env[64020]: DEBUG oslo_concurrency.lockutils [req-4968847b-acc1-46f0-9121-7309e91a41d0 req-42f7a202-20fb-4ec8-a359-a2347600cdf5 service nova] Acquiring lock "refresh_cache-ccffbcd1-09f3-4458-8caa-54980f26da14" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1054.951079] env[64020]: DEBUG oslo_concurrency.lockutils [req-4968847b-acc1-46f0-9121-7309e91a41d0 req-42f7a202-20fb-4ec8-a359-a2347600cdf5 service nova] Acquired lock "refresh_cache-ccffbcd1-09f3-4458-8caa-54980f26da14" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1054.951426] env[64020]: DEBUG nova.network.neutron [req-4968847b-acc1-46f0-9121-7309e91a41d0 req-42f7a202-20fb-4ec8-a359-a2347600cdf5 service nova] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Refreshing network info cache for port 1b3ab288-c471-42f1-867c-364ca86380aa {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1054.952018] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-4968847b-acc1-46f0-9121-7309e91a41d0 req-42f7a202-20fb-4ec8-a359-a2347600cdf5 service nova] Expecting reply to msg debc456bfa594b1ea1b6af8aea5158c8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1054.957356] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.632s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1054.958045] env[64020]: ERROR nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e7d771e4-a10e-4178-af1b-7325b18e280e, please check neutron logs for more information. [ 1054.958045] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Traceback (most recent call last): [ 1054.958045] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1054.958045] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] self.driver.spawn(context, instance, image_meta, [ 1054.958045] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1054.958045] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1054.958045] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1054.958045] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] vm_ref = self.build_virtual_machine(instance, [ 1054.958045] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1054.958045] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] vif_infos = vmwarevif.get_vif_info(self._session, [ 1054.958045] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] for vif in network_info: [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] return self._sync_wrapper(fn, *args, **kwargs) [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] self.wait() [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] self[:] = self._gt.wait() [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] return self._exit_event.wait() [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] current.throw(*self._exc) [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1054.958471] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] result = function(*args, **kwargs) [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] return func(*args, **kwargs) [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] raise e [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] nwinfo = self.network_api.allocate_for_instance( [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] created_port_ids = self._update_ports_for_instance( [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] with excutils.save_and_reraise_exception(): [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] self.force_reraise() [ 1054.958878] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1054.959435] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] raise self.value [ 1054.959435] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1054.959435] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] updated_port = self._update_port( [ 1054.959435] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1054.959435] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] _ensure_no_port_binding_failure(port) [ 1054.959435] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1054.959435] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] raise exception.PortBindingFailed(port_id=port['id']) [ 1054.959435] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] nova.exception.PortBindingFailed: Binding failed for port e7d771e4-a10e-4178-af1b-7325b18e280e, please check neutron logs for more information. [ 1054.959435] env[64020]: ERROR nova.compute.manager [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] [ 1054.960496] env[64020]: DEBUG nova.compute.utils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Binding failed for port e7d771e4-a10e-4178-af1b-7325b18e280e, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1054.962323] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg debc456bfa594b1ea1b6af8aea5158c8 [ 1054.963180] env[64020]: DEBUG nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Build of instance f1785140-ebf1-4f2e-b35b-573d55a3e3a0 was re-scheduled: Binding failed for port e7d771e4-a10e-4178-af1b-7325b18e280e, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1054.963829] env[64020]: DEBUG nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1054.964283] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Acquiring lock "refresh_cache-f1785140-ebf1-4f2e-b35b-573d55a3e3a0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1054.964555] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Acquired lock "refresh_cache-f1785140-ebf1-4f2e-b35b-573d55a3e3a0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1054.964821] env[64020]: DEBUG nova.network.neutron [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1054.965453] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 70c111a9bbdb4672be2df3abb487dc50 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1054.977967] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 70c111a9bbdb4672be2df3abb487dc50 [ 1055.004154] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "b46eab3d-f824-419d-822a-789618584907" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1055.004661] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "b46eab3d-f824-419d-822a-789618584907" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1055.005222] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg fd9b075875b346e98ad26f6ae8518485 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1055.013600] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd9b075875b346e98ad26f6ae8518485 [ 1055.139183] env[64020]: ERROR nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1b3ab288-c471-42f1-867c-364ca86380aa, please check neutron logs for more information. [ 1055.139183] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1055.139183] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1055.139183] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1055.139183] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1055.139183] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1055.139183] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1055.139183] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1055.139183] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1055.139183] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1055.139183] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1055.139183] env[64020]: ERROR nova.compute.manager raise self.value [ 1055.139183] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1055.139183] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1055.139183] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1055.139183] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1055.139825] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1055.139825] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1055.139825] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1b3ab288-c471-42f1-867c-364ca86380aa, please check neutron logs for more information. [ 1055.139825] env[64020]: ERROR nova.compute.manager [ 1055.140208] env[64020]: Traceback (most recent call last): [ 1055.140313] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1055.140313] env[64020]: listener.cb(fileno) [ 1055.140414] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1055.140414] env[64020]: result = function(*args, **kwargs) [ 1055.140493] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1055.140493] env[64020]: return func(*args, **kwargs) [ 1055.140571] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1055.140571] env[64020]: raise e [ 1055.140648] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1055.140648] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1055.140733] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1055.140733] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1055.140813] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1055.140813] env[64020]: with excutils.save_and_reraise_exception(): [ 1055.140891] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1055.140891] env[64020]: self.force_reraise() [ 1055.140969] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1055.140969] env[64020]: raise self.value [ 1055.141051] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1055.141051] env[64020]: updated_port = self._update_port( [ 1055.141129] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1055.141129] env[64020]: _ensure_no_port_binding_failure(port) [ 1055.141310] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1055.141310] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1055.141413] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 1b3ab288-c471-42f1-867c-364ca86380aa, please check neutron logs for more information. [ 1055.141468] env[64020]: Removing descriptor: 16 [ 1055.351160] env[64020]: DEBUG nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1055.376688] env[64020]: DEBUG nova.virt.hardware [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1055.376850] env[64020]: DEBUG nova.virt.hardware [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1055.377038] env[64020]: DEBUG nova.virt.hardware [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1055.377245] env[64020]: DEBUG nova.virt.hardware [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1055.377510] env[64020]: DEBUG nova.virt.hardware [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1055.377890] env[64020]: DEBUG nova.virt.hardware [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1055.378139] env[64020]: DEBUG nova.virt.hardware [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1055.378302] env[64020]: DEBUG nova.virt.hardware [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1055.378518] env[64020]: DEBUG nova.virt.hardware [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1055.378712] env[64020]: DEBUG nova.virt.hardware [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1055.378885] env[64020]: DEBUG nova.virt.hardware [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1055.379741] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3a791b32-1c10-4fe8-b39f-6b9f52b0b8b5 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1055.387827] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab80b2da-798d-4693-9a97-4518fdf20629 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1055.404674] env[64020]: ERROR nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1b3ab288-c471-42f1-867c-364ca86380aa, please check neutron logs for more information. [ 1055.404674] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Traceback (most recent call last): [ 1055.404674] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1055.404674] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] yield resources [ 1055.404674] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1055.404674] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] self.driver.spawn(context, instance, image_meta, [ 1055.404674] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1055.404674] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1055.404674] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1055.404674] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] vm_ref = self.build_virtual_machine(instance, [ 1055.404674] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] vif_infos = vmwarevif.get_vif_info(self._session, [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] for vif in network_info: [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] return self._sync_wrapper(fn, *args, **kwargs) [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] self.wait() [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] self[:] = self._gt.wait() [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] return self._exit_event.wait() [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1055.405071] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] current.throw(*self._exc) [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] result = function(*args, **kwargs) [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] return func(*args, **kwargs) [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] raise e [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] nwinfo = self.network_api.allocate_for_instance( [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] created_port_ids = self._update_ports_for_instance( [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] with excutils.save_and_reraise_exception(): [ 1055.405466] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1055.405849] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] self.force_reraise() [ 1055.405849] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1055.405849] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] raise self.value [ 1055.405849] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1055.405849] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] updated_port = self._update_port( [ 1055.405849] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1055.405849] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] _ensure_no_port_binding_failure(port) [ 1055.405849] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1055.405849] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] raise exception.PortBindingFailed(port_id=port['id']) [ 1055.405849] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] nova.exception.PortBindingFailed: Binding failed for port 1b3ab288-c471-42f1-867c-364ca86380aa, please check neutron logs for more information. [ 1055.405849] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] [ 1055.405849] env[64020]: INFO nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Terminating instance [ 1055.407035] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "refresh_cache-ccffbcd1-09f3-4458-8caa-54980f26da14" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1055.472775] env[64020]: DEBUG nova.network.neutron [req-4968847b-acc1-46f0-9121-7309e91a41d0 req-42f7a202-20fb-4ec8-a359-a2347600cdf5 service nova] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1055.499181] env[64020]: DEBUG nova.network.neutron [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1055.507613] env[64020]: DEBUG nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1055.509449] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg fe565a1d199240c2b9099769e311da5e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1055.542509] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe565a1d199240c2b9099769e311da5e [ 1055.764837] env[64020]: DEBUG nova.network.neutron [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1055.765376] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 9a4c89b6df584ab29a3839135e45875b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1055.773419] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9a4c89b6df584ab29a3839135e45875b [ 1055.799759] env[64020]: DEBUG nova.network.neutron [req-4968847b-acc1-46f0-9121-7309e91a41d0 req-42f7a202-20fb-4ec8-a359-a2347600cdf5 service nova] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1055.800525] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-4968847b-acc1-46f0-9121-7309e91a41d0 req-42f7a202-20fb-4ec8-a359-a2347600cdf5 service nova] Expecting reply to msg 915289757ce7463db9df0edd931ad4e3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1055.808389] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 915289757ce7463db9df0edd931ad4e3 [ 1056.028332] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1056.028590] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1056.030063] env[64020]: INFO nova.compute.claims [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1056.031617] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 0c93355dc23c4310a304695ed38f30b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1056.069214] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c93355dc23c4310a304695ed38f30b3 [ 1056.270999] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Releasing lock "refresh_cache-f1785140-ebf1-4f2e-b35b-573d55a3e3a0" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1056.270999] env[64020]: DEBUG nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1056.270999] env[64020]: DEBUG nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1056.270999] env[64020]: DEBUG nova.network.neutron [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1056.286093] env[64020]: DEBUG nova.network.neutron [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1056.286093] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 64eb762ddb274b55ab7dd885a245f705 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1056.292199] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64eb762ddb274b55ab7dd885a245f705 [ 1056.302497] env[64020]: DEBUG oslo_concurrency.lockutils [req-4968847b-acc1-46f0-9121-7309e91a41d0 req-42f7a202-20fb-4ec8-a359-a2347600cdf5 service nova] Releasing lock "refresh_cache-ccffbcd1-09f3-4458-8caa-54980f26da14" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1056.303000] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquired lock "refresh_cache-ccffbcd1-09f3-4458-8caa-54980f26da14" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1056.303347] env[64020]: DEBUG nova.network.neutron [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1056.303931] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg ac7ea307bf5a4ad9b3d5225235bee766 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1056.310984] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ac7ea307bf5a4ad9b3d5225235bee766 [ 1056.536930] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg b5e56aee67394bddb9b6c592b19f8475 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1056.545063] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b5e56aee67394bddb9b6c592b19f8475 [ 1056.788542] env[64020]: DEBUG nova.network.neutron [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1056.788542] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg f21d01d8152a40b58624f04d0eee66ec in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1056.797056] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f21d01d8152a40b58624f04d0eee66ec [ 1056.825767] env[64020]: DEBUG nova.network.neutron [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1056.911826] env[64020]: DEBUG nova.network.neutron [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1056.911826] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 7311f93a3c8b4283a30027a841ff261e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1056.920472] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7311f93a3c8b4283a30027a841ff261e [ 1056.981366] env[64020]: DEBUG nova.compute.manager [req-ebd59942-f2d0-4293-a5f0-9c6911d41dc3 req-4099dd76-591d-4729-a7d8-8acb550f1612 service nova] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Received event network-vif-deleted-1b3ab288-c471-42f1-867c-364ca86380aa {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1057.099720] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35520b15-6e11-4e2d-a5f2-6c475ff88319 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1057.108819] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbe4bd2b-b357-4422-aad3-fd0fe00df817 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1057.141761] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e3a8c26-33e1-4218-85f3-c830f6cd28c8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1057.149193] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f26d1e1e-5087-4c27-ad96-d1e754904aea {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1057.163652] env[64020]: DEBUG nova.compute.provider_tree [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1057.164326] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 69a1cbb3dad94bdfa2b95ad0822c42f1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1057.172303] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69a1cbb3dad94bdfa2b95ad0822c42f1 [ 1057.293925] env[64020]: INFO nova.compute.manager [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] [instance: f1785140-ebf1-4f2e-b35b-573d55a3e3a0] Took 1.02 seconds to deallocate network for instance. [ 1057.296463] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg de2fb4c6af0347f797dca7daaa348d56 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1057.334362] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de2fb4c6af0347f797dca7daaa348d56 [ 1057.416031] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Releasing lock "refresh_cache-ccffbcd1-09f3-4458-8caa-54980f26da14" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1057.416031] env[64020]: DEBUG nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1057.416031] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1057.416031] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-343d2f9d-b890-471f-8c2b-59562e317b42 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1057.424714] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfad8990-36cc-42a5-94fa-eb870b89c059 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1057.445580] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ccffbcd1-09f3-4458-8caa-54980f26da14 could not be found. [ 1057.445794] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1057.445971] env[64020]: INFO nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1057.446205] env[64020]: DEBUG oslo.service.loopingcall [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1057.446473] env[64020]: DEBUG nova.compute.manager [-] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1057.446553] env[64020]: DEBUG nova.network.neutron [-] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1057.468028] env[64020]: DEBUG nova.network.neutron [-] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1057.468510] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 96022dd38e92464e9540c0ce3ffc4acb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1057.475594] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 96022dd38e92464e9540c0ce3ffc4acb [ 1057.667226] env[64020]: DEBUG nova.scheduler.client.report [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1057.669682] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 614802dfa1a04e84a10db979747bad60 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1057.682654] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 614802dfa1a04e84a10db979747bad60 [ 1057.802078] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg a459c58c4f38427884aa9c759da9982c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1057.844240] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a459c58c4f38427884aa9c759da9982c [ 1057.970447] env[64020]: DEBUG nova.network.neutron [-] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1057.970918] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 94a0d02051d44d49a338dd303ddbc833 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1057.979243] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94a0d02051d44d49a338dd303ddbc833 [ 1058.172410] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.144s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1058.172915] env[64020]: DEBUG nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1058.174596] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 6341adff4e32408f9b29d268613fa8b6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1058.205478] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6341adff4e32408f9b29d268613fa8b6 [ 1058.326286] env[64020]: INFO nova.scheduler.client.report [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Deleted allocations for instance f1785140-ebf1-4f2e-b35b-573d55a3e3a0 [ 1058.332353] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Expecting reply to msg 6f654b6a6aa24bbc91a73b546979108d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1058.345884] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f654b6a6aa24bbc91a73b546979108d [ 1058.473775] env[64020]: INFO nova.compute.manager [-] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Took 1.03 seconds to deallocate network for instance. [ 1058.477980] env[64020]: DEBUG nova.compute.claims [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1058.478167] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1058.478382] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1058.480136] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 05f636f2e17043ae96bc6c5f7027fd4e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1058.512510] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05f636f2e17043ae96bc6c5f7027fd4e [ 1058.677624] env[64020]: DEBUG nova.compute.utils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1058.678273] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 54919c68093242e4875252845eeefed3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1058.679231] env[64020]: DEBUG nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1058.680020] env[64020]: DEBUG nova.network.neutron [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1058.688685] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54919c68093242e4875252845eeefed3 [ 1058.734005] env[64020]: DEBUG nova.policy [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd496b47ef9db47fa97323baf6bbd63ed', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f1fb1191ee044c4f921f5b7935f0109d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1058.834392] env[64020]: DEBUG oslo_concurrency.lockutils [None req-dcf0e018-1107-44e8-b228-5cdb4783f38b tempest-ServerActionsTestOtherA-356726708 tempest-ServerActionsTestOtherA-356726708-project-member] Lock "f1785140-ebf1-4f2e-b35b-573d55a3e3a0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 14.111s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1059.004798] env[64020]: DEBUG nova.network.neutron [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Successfully created port: eab402f3-4651-4980-a6e5-86ec594dbba4 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1059.019522] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9877d293-e483-4d73-b2f5-7403a0abe7cc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1059.026981] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-454dd349-88df-4daa-8f7c-b2344f81a33f {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1059.057348] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf55d564-7e25-46a3-83f2-2240fb944bcc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1059.064804] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1070f28f-2f03-4757-bd39-c2be0d253483 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1059.078717] env[64020]: DEBUG nova.compute.provider_tree [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1059.079222] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 82f25e906f6142009c249775411f5462 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1059.091170] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82f25e906f6142009c249775411f5462 [ 1059.182975] env[64020]: DEBUG nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1059.184746] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg fdbb687f6e7745fa9fa5bebd82d715be in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1059.234982] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fdbb687f6e7745fa9fa5bebd82d715be [ 1059.581664] env[64020]: DEBUG nova.scheduler.client.report [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1059.583996] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 63d57e5583f548b89413a5c3465c9a9d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1059.597862] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 63d57e5583f548b89413a5c3465c9a9d [ 1059.689835] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 43eb7362a91744dc846e19a3d468ddb8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1059.730508] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43eb7362a91744dc846e19a3d468ddb8 [ 1060.086896] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.608s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1060.087486] env[64020]: ERROR nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1b3ab288-c471-42f1-867c-364ca86380aa, please check neutron logs for more information. [ 1060.087486] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Traceback (most recent call last): [ 1060.087486] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1060.087486] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] self.driver.spawn(context, instance, image_meta, [ 1060.087486] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1060.087486] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1060.087486] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1060.087486] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] vm_ref = self.build_virtual_machine(instance, [ 1060.087486] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1060.087486] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] vif_infos = vmwarevif.get_vif_info(self._session, [ 1060.087486] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] for vif in network_info: [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] return self._sync_wrapper(fn, *args, **kwargs) [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] self.wait() [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] self[:] = self._gt.wait() [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] return self._exit_event.wait() [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] current.throw(*self._exc) [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1060.087907] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] result = function(*args, **kwargs) [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] return func(*args, **kwargs) [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] raise e [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] nwinfo = self.network_api.allocate_for_instance( [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] created_port_ids = self._update_ports_for_instance( [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] with excutils.save_and_reraise_exception(): [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] self.force_reraise() [ 1060.088334] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1060.088785] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] raise self.value [ 1060.088785] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1060.088785] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] updated_port = self._update_port( [ 1060.088785] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1060.088785] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] _ensure_no_port_binding_failure(port) [ 1060.088785] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1060.088785] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] raise exception.PortBindingFailed(port_id=port['id']) [ 1060.088785] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] nova.exception.PortBindingFailed: Binding failed for port 1b3ab288-c471-42f1-867c-364ca86380aa, please check neutron logs for more information. [ 1060.088785] env[64020]: ERROR nova.compute.manager [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] [ 1060.088785] env[64020]: DEBUG nova.compute.utils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Binding failed for port 1b3ab288-c471-42f1-867c-364ca86380aa, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1060.092195] env[64020]: DEBUG nova.compute.manager [req-4bce6a37-64bb-44f8-a3bb-fcddd54548bb req-189ca6f0-3c26-4009-ba41-042ba3b1ac0c service nova] [instance: b46eab3d-f824-419d-822a-789618584907] Received event network-changed-eab402f3-4651-4980-a6e5-86ec594dbba4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1060.092195] env[64020]: DEBUG nova.compute.manager [req-4bce6a37-64bb-44f8-a3bb-fcddd54548bb req-189ca6f0-3c26-4009-ba41-042ba3b1ac0c service nova] [instance: b46eab3d-f824-419d-822a-789618584907] Refreshing instance network info cache due to event network-changed-eab402f3-4651-4980-a6e5-86ec594dbba4. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1060.092195] env[64020]: DEBUG oslo_concurrency.lockutils [req-4bce6a37-64bb-44f8-a3bb-fcddd54548bb req-189ca6f0-3c26-4009-ba41-042ba3b1ac0c service nova] Acquiring lock "refresh_cache-b46eab3d-f824-419d-822a-789618584907" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1060.092195] env[64020]: DEBUG oslo_concurrency.lockutils [req-4bce6a37-64bb-44f8-a3bb-fcddd54548bb req-189ca6f0-3c26-4009-ba41-042ba3b1ac0c service nova] Acquired lock "refresh_cache-b46eab3d-f824-419d-822a-789618584907" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1060.092195] env[64020]: DEBUG nova.network.neutron [req-4bce6a37-64bb-44f8-a3bb-fcddd54548bb req-189ca6f0-3c26-4009-ba41-042ba3b1ac0c service nova] [instance: b46eab3d-f824-419d-822a-789618584907] Refreshing network info cache for port eab402f3-4651-4980-a6e5-86ec594dbba4 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1060.092422] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-4bce6a37-64bb-44f8-a3bb-fcddd54548bb req-189ca6f0-3c26-4009-ba41-042ba3b1ac0c service nova] Expecting reply to msg 2ba26b851b314990b4931ba9b942e7ae in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1060.093562] env[64020]: DEBUG nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Build of instance ccffbcd1-09f3-4458-8caa-54980f26da14 was re-scheduled: Binding failed for port 1b3ab288-c471-42f1-867c-364ca86380aa, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1060.093722] env[64020]: DEBUG nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1060.094004] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquiring lock "refresh_cache-ccffbcd1-09f3-4458-8caa-54980f26da14" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1060.094077] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Acquired lock "refresh_cache-ccffbcd1-09f3-4458-8caa-54980f26da14" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1060.094201] env[64020]: DEBUG nova.network.neutron [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1060.094556] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg af9c4c8967574bbdaf098264cb4cbb45 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1060.111790] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ba26b851b314990b4931ba9b942e7ae [ 1060.111901] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af9c4c8967574bbdaf098264cb4cbb45 [ 1060.194800] env[64020]: DEBUG nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1060.220406] env[64020]: DEBUG nova.virt.hardware [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1060.221039] env[64020]: DEBUG nova.virt.hardware [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1060.221039] env[64020]: DEBUG nova.virt.hardware [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1060.221039] env[64020]: DEBUG nova.virt.hardware [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1060.221176] env[64020]: DEBUG nova.virt.hardware [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1060.221243] env[64020]: DEBUG nova.virt.hardware [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1060.221480] env[64020]: DEBUG nova.virt.hardware [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1060.221632] env[64020]: DEBUG nova.virt.hardware [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1060.221790] env[64020]: DEBUG nova.virt.hardware [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1060.221964] env[64020]: DEBUG nova.virt.hardware [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1060.222102] env[64020]: DEBUG nova.virt.hardware [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1060.222952] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-999051c1-96c4-48cc-a293-bc6f92fcf874 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1060.231237] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc83c221-62ee-483e-87d9-30768fb48d56 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1060.273211] env[64020]: ERROR nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port eab402f3-4651-4980-a6e5-86ec594dbba4, please check neutron logs for more information. [ 1060.273211] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1060.273211] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1060.273211] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1060.273211] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1060.273211] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1060.273211] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1060.273211] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1060.273211] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1060.273211] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1060.273211] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1060.273211] env[64020]: ERROR nova.compute.manager raise self.value [ 1060.273211] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1060.273211] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1060.273211] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1060.273211] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1060.273834] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1060.273834] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1060.273834] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port eab402f3-4651-4980-a6e5-86ec594dbba4, please check neutron logs for more information. [ 1060.273834] env[64020]: ERROR nova.compute.manager [ 1060.273834] env[64020]: Traceback (most recent call last): [ 1060.273834] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1060.273834] env[64020]: listener.cb(fileno) [ 1060.273834] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1060.273834] env[64020]: result = function(*args, **kwargs) [ 1060.273834] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1060.273834] env[64020]: return func(*args, **kwargs) [ 1060.273834] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1060.273834] env[64020]: raise e [ 1060.273834] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1060.273834] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1060.273834] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1060.273834] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1060.273834] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1060.273834] env[64020]: with excutils.save_and_reraise_exception(): [ 1060.273834] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1060.273834] env[64020]: self.force_reraise() [ 1060.273834] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1060.273834] env[64020]: raise self.value [ 1060.273834] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1060.273834] env[64020]: updated_port = self._update_port( [ 1060.273834] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1060.273834] env[64020]: _ensure_no_port_binding_failure(port) [ 1060.273834] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1060.273834] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1060.274859] env[64020]: nova.exception.PortBindingFailed: Binding failed for port eab402f3-4651-4980-a6e5-86ec594dbba4, please check neutron logs for more information. [ 1060.274859] env[64020]: Removing descriptor: 16 [ 1060.274859] env[64020]: ERROR nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port eab402f3-4651-4980-a6e5-86ec594dbba4, please check neutron logs for more information. [ 1060.274859] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] Traceback (most recent call last): [ 1060.274859] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1060.274859] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] yield resources [ 1060.274859] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1060.274859] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] self.driver.spawn(context, instance, image_meta, [ 1060.274859] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1060.274859] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1060.274859] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1060.274859] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] vm_ref = self.build_virtual_machine(instance, [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] vif_infos = vmwarevif.get_vif_info(self._session, [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] for vif in network_info: [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] return self._sync_wrapper(fn, *args, **kwargs) [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] self.wait() [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] self[:] = self._gt.wait() [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] return self._exit_event.wait() [ 1060.275232] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] result = hub.switch() [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] return self.greenlet.switch() [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] result = function(*args, **kwargs) [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] return func(*args, **kwargs) [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] raise e [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] nwinfo = self.network_api.allocate_for_instance( [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1060.275640] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] created_port_ids = self._update_ports_for_instance( [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] with excutils.save_and_reraise_exception(): [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] self.force_reraise() [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] raise self.value [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] updated_port = self._update_port( [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] _ensure_no_port_binding_failure(port) [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1060.276115] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] raise exception.PortBindingFailed(port_id=port['id']) [ 1060.276470] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] nova.exception.PortBindingFailed: Binding failed for port eab402f3-4651-4980-a6e5-86ec594dbba4, please check neutron logs for more information. [ 1060.276470] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] [ 1060.276470] env[64020]: INFO nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Terminating instance [ 1060.277839] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-b46eab3d-f824-419d-822a-789618584907" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1060.624680] env[64020]: DEBUG nova.network.neutron [req-4bce6a37-64bb-44f8-a3bb-fcddd54548bb req-189ca6f0-3c26-4009-ba41-042ba3b1ac0c service nova] [instance: b46eab3d-f824-419d-822a-789618584907] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1060.640437] env[64020]: DEBUG nova.network.neutron [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1060.720700] env[64020]: DEBUG nova.network.neutron [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1060.721212] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg a337e13daa53454e8dc9b09640ae2b56 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1060.731381] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a337e13daa53454e8dc9b09640ae2b56 [ 1060.804295] env[64020]: DEBUG nova.network.neutron [req-4bce6a37-64bb-44f8-a3bb-fcddd54548bb req-189ca6f0-3c26-4009-ba41-042ba3b1ac0c service nova] [instance: b46eab3d-f824-419d-822a-789618584907] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1060.804757] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-4bce6a37-64bb-44f8-a3bb-fcddd54548bb req-189ca6f0-3c26-4009-ba41-042ba3b1ac0c service nova] Expecting reply to msg 60a0d6a3acbb4a91a97fa9cd5a69a622 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1060.813771] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60a0d6a3acbb4a91a97fa9cd5a69a622 [ 1061.224202] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Releasing lock "refresh_cache-ccffbcd1-09f3-4458-8caa-54980f26da14" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1061.224546] env[64020]: DEBUG nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1061.224630] env[64020]: DEBUG nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1061.225085] env[64020]: DEBUG nova.network.neutron [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1061.242254] env[64020]: DEBUG nova.network.neutron [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1061.242933] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg cd97245f9dd644fb89d5fa5dbc5e973e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1061.253096] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd97245f9dd644fb89d5fa5dbc5e973e [ 1061.307569] env[64020]: DEBUG oslo_concurrency.lockutils [req-4bce6a37-64bb-44f8-a3bb-fcddd54548bb req-189ca6f0-3c26-4009-ba41-042ba3b1ac0c service nova] Releasing lock "refresh_cache-b46eab3d-f824-419d-822a-789618584907" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1061.308103] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-b46eab3d-f824-419d-822a-789618584907" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1061.308423] env[64020]: DEBUG nova.network.neutron [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1061.309968] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 405be8c85e374a7b9f3b724738f8f1ec in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1061.326808] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 405be8c85e374a7b9f3b724738f8f1ec [ 1061.750681] env[64020]: DEBUG nova.network.neutron [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1061.751210] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 2b48c5192cdf4b62972e95a2e08ca83f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1061.768366] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b48c5192cdf4b62972e95a2e08ca83f [ 1061.830226] env[64020]: DEBUG nova.network.neutron [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1061.925448] env[64020]: DEBUG nova.network.neutron [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1061.925957] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 354f075a1dc94b44a2c67b817e3dce31 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1061.935033] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 354f075a1dc94b44a2c67b817e3dce31 [ 1062.119234] env[64020]: DEBUG nova.compute.manager [req-0a054bc6-df43-4d99-85b6-592496e4b83b req-20a96d93-d8c3-49e2-86f4-5e66ff07d196 service nova] [instance: b46eab3d-f824-419d-822a-789618584907] Received event network-vif-deleted-eab402f3-4651-4980-a6e5-86ec594dbba4 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1062.263129] env[64020]: INFO nova.compute.manager [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] [instance: ccffbcd1-09f3-4458-8caa-54980f26da14] Took 1.04 seconds to deallocate network for instance. [ 1062.264893] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg d55620ab5ea14859a3cfc48cea815baa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1062.305460] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d55620ab5ea14859a3cfc48cea815baa [ 1062.375454] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Acquiring lock "dea85e5f-ed81-41d2-bcea-906ba68c132b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1062.375702] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Lock "dea85e5f-ed81-41d2-bcea-906ba68c132b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1062.377183] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 5dee2159964e4c0e89db80fa710a6bc9 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1062.386535] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5dee2159964e4c0e89db80fa710a6bc9 [ 1062.428335] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-b46eab3d-f824-419d-822a-789618584907" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1062.428776] env[64020]: DEBUG nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1062.428961] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1062.429329] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1c72db67-8283-4769-89a2-2a51f5177e2c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1062.438089] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bee105bd-4549-432d-8a27-13dacd2acddc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1062.459530] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b46eab3d-f824-419d-822a-789618584907 could not be found. [ 1062.459806] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1062.459996] env[64020]: INFO nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1062.460384] env[64020]: DEBUG oslo.service.loopingcall [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1062.460630] env[64020]: DEBUG nova.compute.manager [-] [instance: b46eab3d-f824-419d-822a-789618584907] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1062.460719] env[64020]: DEBUG nova.network.neutron [-] [instance: b46eab3d-f824-419d-822a-789618584907] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1062.490256] env[64020]: DEBUG nova.network.neutron [-] [instance: b46eab3d-f824-419d-822a-789618584907] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1062.490772] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 850a9010414c4453827da6853f274b7c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1062.498137] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 850a9010414c4453827da6853f274b7c [ 1062.771124] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 447e30571d914dd8964ff95f2f76d195 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1062.808605] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 447e30571d914dd8964ff95f2f76d195 [ 1062.879244] env[64020]: DEBUG nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1062.880944] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg ce5d1d20fd454a0f94dae84ed9a82c83 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1062.911713] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce5d1d20fd454a0f94dae84ed9a82c83 [ 1062.993822] env[64020]: DEBUG nova.network.neutron [-] [instance: b46eab3d-f824-419d-822a-789618584907] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1062.994319] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bf779f2e8246484fb8c77c2e43c0bf1f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1063.002699] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf779f2e8246484fb8c77c2e43c0bf1f [ 1063.295136] env[64020]: INFO nova.scheduler.client.report [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Deleted allocations for instance ccffbcd1-09f3-4458-8caa-54980f26da14 [ 1063.316267] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Expecting reply to msg 08910f2048e34902b41c5cb0d58e27ec in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1063.330023] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08910f2048e34902b41c5cb0d58e27ec [ 1063.397848] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1063.398253] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1063.400151] env[64020]: INFO nova.compute.claims [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1063.401812] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg cefe328691da41e19ea38bc2fa689561 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1063.440060] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cefe328691da41e19ea38bc2fa689561 [ 1063.497626] env[64020]: INFO nova.compute.manager [-] [instance: b46eab3d-f824-419d-822a-789618584907] Took 1.04 seconds to deallocate network for instance. [ 1063.499847] env[64020]: DEBUG nova.compute.claims [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1063.500039] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1063.819450] env[64020]: DEBUG oslo_concurrency.lockutils [None req-25d7bce5-2415-4046-ab6f-779dbe06eb7b tempest-ServerDiskConfigTestJSON-865109624 tempest-ServerDiskConfigTestJSON-865109624-project-member] Lock "ccffbcd1-09f3-4458-8caa-54980f26da14" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 13.973s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1063.905551] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 6184266ff3054d87913d577e0029e564 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1063.913371] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6184266ff3054d87913d577e0029e564 [ 1064.443534] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a16a3d0f-ec1f-4511-ac42-929bb8336464 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1064.451205] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19bfd5e7-e631-43b9-a7b9-e1ff946320dc {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1064.481589] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0da28221-078a-46b2-b8b5-133c7f6260f1 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1064.487986] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e804c4b-6c75-4115-ba3f-3b57c49c75a3 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1064.500691] env[64020]: DEBUG nova.compute.provider_tree [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1064.501148] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 7146ee95017740b69190d73a2d2ef91b in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1064.507602] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7146ee95017740b69190d73a2d2ef91b [ 1065.003458] env[64020]: DEBUG nova.scheduler.client.report [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1065.009241] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 64e7d33386dc4385bc47d5019f51c7a7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1065.018009] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64e7d33386dc4385bc47d5019f51c7a7 [ 1065.508228] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.110s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1065.508768] env[64020]: DEBUG nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1065.510406] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg fcf8421eb8324f26a05a7d3e07c511c4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1065.511433] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 2.011s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1065.513176] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg eae0e231dbbf4292a78934790f6117e6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1065.550401] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eae0e231dbbf4292a78934790f6117e6 [ 1065.550913] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fcf8421eb8324f26a05a7d3e07c511c4 [ 1066.016782] env[64020]: DEBUG nova.compute.utils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1066.017400] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 0fb5e2a2c46a471d8a959e88ffc82ce8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1066.018443] env[64020]: DEBUG nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1066.019844] env[64020]: DEBUG nova.network.neutron [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1066.030715] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0fb5e2a2c46a471d8a959e88ffc82ce8 [ 1066.066582] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf7bb7eb-58d6-407e-bf8e-d5dec60bfa6e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1066.072498] env[64020]: DEBUG nova.policy [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'aad25c1b15514c42b679ad8c4b59c034', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '62a66aa26e0e4462a908d01221860ec8', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1066.077798] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f750a46a-0949-4cc0-8126-b1dc8eafc986 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1066.110715] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20a8c128-d095-4465-beb0-4b2b1c0f0635 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1066.117793] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35091876-cf2d-436a-9925-5229b1bfa869 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1066.131305] env[64020]: DEBUG nova.compute.provider_tree [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1066.131843] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg fac8717c1c304b8ebb902373986c2e93 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1066.140123] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fac8717c1c304b8ebb902373986c2e93 [ 1066.324799] env[64020]: DEBUG nova.network.neutron [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Successfully created port: 10ad0546-c51a-459e-9f38-34479abd3eb2 {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1066.521720] env[64020]: DEBUG nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1066.523956] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 7f0dff3606224dab9ba6ae475e95de14 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1066.557747] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f0dff3606224dab9ba6ae475e95de14 [ 1066.635064] env[64020]: DEBUG nova.scheduler.client.report [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1066.637363] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg d1cf6a3144ad4441a2368b8866c941f4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1066.650637] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1cf6a3144ad4441a2368b8866c941f4 [ 1066.928352] env[64020]: DEBUG nova.compute.manager [req-ef18949c-96a9-4bd6-87a8-2629423e0679 req-46f6b411-107c-4118-913f-d0b71f7f32b6 service nova] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Received event network-changed-10ad0546-c51a-459e-9f38-34479abd3eb2 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1066.928560] env[64020]: DEBUG nova.compute.manager [req-ef18949c-96a9-4bd6-87a8-2629423e0679 req-46f6b411-107c-4118-913f-d0b71f7f32b6 service nova] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Refreshing instance network info cache due to event network-changed-10ad0546-c51a-459e-9f38-34479abd3eb2. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1066.928773] env[64020]: DEBUG oslo_concurrency.lockutils [req-ef18949c-96a9-4bd6-87a8-2629423e0679 req-46f6b411-107c-4118-913f-d0b71f7f32b6 service nova] Acquiring lock "refresh_cache-dea85e5f-ed81-41d2-bcea-906ba68c132b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1066.928929] env[64020]: DEBUG oslo_concurrency.lockutils [req-ef18949c-96a9-4bd6-87a8-2629423e0679 req-46f6b411-107c-4118-913f-d0b71f7f32b6 service nova] Acquired lock "refresh_cache-dea85e5f-ed81-41d2-bcea-906ba68c132b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1066.929055] env[64020]: DEBUG nova.network.neutron [req-ef18949c-96a9-4bd6-87a8-2629423e0679 req-46f6b411-107c-4118-913f-d0b71f7f32b6 service nova] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Refreshing network info cache for port 10ad0546-c51a-459e-9f38-34479abd3eb2 {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1066.929463] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-ef18949c-96a9-4bd6-87a8-2629423e0679 req-46f6b411-107c-4118-913f-d0b71f7f32b6 service nova] Expecting reply to msg e03b2465696845fcb6e2144fbd72da7e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1066.936469] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e03b2465696845fcb6e2144fbd72da7e [ 1067.031007] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 7f52c82f8e3648939078ff3c8a795558 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1067.067088] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f52c82f8e3648939078ff3c8a795558 [ 1067.140418] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.629s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1067.140982] env[64020]: ERROR nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port eab402f3-4651-4980-a6e5-86ec594dbba4, please check neutron logs for more information. [ 1067.140982] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] Traceback (most recent call last): [ 1067.140982] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1067.140982] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] self.driver.spawn(context, instance, image_meta, [ 1067.140982] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1067.140982] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1067.140982] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1067.140982] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] vm_ref = self.build_virtual_machine(instance, [ 1067.140982] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1067.140982] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] vif_infos = vmwarevif.get_vif_info(self._session, [ 1067.140982] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] for vif in network_info: [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] return self._sync_wrapper(fn, *args, **kwargs) [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] self.wait() [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] self[:] = self._gt.wait() [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] return self._exit_event.wait() [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] result = hub.switch() [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1067.141487] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] return self.greenlet.switch() [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] result = function(*args, **kwargs) [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] return func(*args, **kwargs) [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] raise e [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] nwinfo = self.network_api.allocate_for_instance( [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] created_port_ids = self._update_ports_for_instance( [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] with excutils.save_and_reraise_exception(): [ 1067.141853] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1067.142212] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] self.force_reraise() [ 1067.142212] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1067.142212] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] raise self.value [ 1067.142212] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1067.142212] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] updated_port = self._update_port( [ 1067.142212] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1067.142212] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] _ensure_no_port_binding_failure(port) [ 1067.142212] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1067.142212] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] raise exception.PortBindingFailed(port_id=port['id']) [ 1067.142212] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] nova.exception.PortBindingFailed: Binding failed for port eab402f3-4651-4980-a6e5-86ec594dbba4, please check neutron logs for more information. [ 1067.142212] env[64020]: ERROR nova.compute.manager [instance: b46eab3d-f824-419d-822a-789618584907] [ 1067.142743] env[64020]: DEBUG nova.compute.utils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Binding failed for port eab402f3-4651-4980-a6e5-86ec594dbba4, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1067.143630] env[64020]: DEBUG nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Build of instance b46eab3d-f824-419d-822a-789618584907 was re-scheduled: Binding failed for port eab402f3-4651-4980-a6e5-86ec594dbba4, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1067.144083] env[64020]: DEBUG nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1067.144313] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquiring lock "refresh_cache-b46eab3d-f824-419d-822a-789618584907" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1067.144501] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Acquired lock "refresh_cache-b46eab3d-f824-419d-822a-789618584907" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1067.144670] env[64020]: DEBUG nova.network.neutron [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1067.145081] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg e2e4d5b6b0634c0583f732bfe477d2dc in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1067.152640] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e2e4d5b6b0634c0583f732bfe477d2dc [ 1067.359868] env[64020]: ERROR nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 10ad0546-c51a-459e-9f38-34479abd3eb2, please check neutron logs for more information. [ 1067.359868] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1067.359868] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1067.359868] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1067.359868] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1067.359868] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1067.359868] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1067.359868] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1067.359868] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1067.359868] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1067.359868] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1067.359868] env[64020]: ERROR nova.compute.manager raise self.value [ 1067.359868] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1067.359868] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1067.359868] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1067.359868] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1067.360397] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1067.360397] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1067.360397] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 10ad0546-c51a-459e-9f38-34479abd3eb2, please check neutron logs for more information. [ 1067.360397] env[64020]: ERROR nova.compute.manager [ 1067.360397] env[64020]: Traceback (most recent call last): [ 1067.360397] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1067.360397] env[64020]: listener.cb(fileno) [ 1067.360397] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1067.360397] env[64020]: result = function(*args, **kwargs) [ 1067.360397] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1067.360397] env[64020]: return func(*args, **kwargs) [ 1067.360397] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1067.360397] env[64020]: raise e [ 1067.360397] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1067.360397] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1067.360397] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1067.360397] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1067.360397] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1067.360397] env[64020]: with excutils.save_and_reraise_exception(): [ 1067.360397] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1067.360397] env[64020]: self.force_reraise() [ 1067.360397] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1067.360397] env[64020]: raise self.value [ 1067.360397] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1067.360397] env[64020]: updated_port = self._update_port( [ 1067.360397] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1067.360397] env[64020]: _ensure_no_port_binding_failure(port) [ 1067.360397] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1067.360397] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1067.361221] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 10ad0546-c51a-459e-9f38-34479abd3eb2, please check neutron logs for more information. [ 1067.361221] env[64020]: Removing descriptor: 16 [ 1067.444458] env[64020]: DEBUG nova.network.neutron [req-ef18949c-96a9-4bd6-87a8-2629423e0679 req-46f6b411-107c-4118-913f-d0b71f7f32b6 service nova] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1067.518795] env[64020]: DEBUG nova.network.neutron [req-ef18949c-96a9-4bd6-87a8-2629423e0679 req-46f6b411-107c-4118-913f-d0b71f7f32b6 service nova] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1067.519320] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-ef18949c-96a9-4bd6-87a8-2629423e0679 req-46f6b411-107c-4118-913f-d0b71f7f32b6 service nova] Expecting reply to msg 60af66d7e7cb415e9311532ce06bf06d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1067.527364] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60af66d7e7cb415e9311532ce06bf06d [ 1067.533593] env[64020]: DEBUG nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1067.558319] env[64020]: DEBUG nova.virt.hardware [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1067.558553] env[64020]: DEBUG nova.virt.hardware [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1067.558705] env[64020]: DEBUG nova.virt.hardware [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1067.558882] env[64020]: DEBUG nova.virt.hardware [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1067.559041] env[64020]: DEBUG nova.virt.hardware [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1067.559167] env[64020]: DEBUG nova.virt.hardware [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1067.559367] env[64020]: DEBUG nova.virt.hardware [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1067.559524] env[64020]: DEBUG nova.virt.hardware [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1067.559688] env[64020]: DEBUG nova.virt.hardware [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1067.559844] env[64020]: DEBUG nova.virt.hardware [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1067.560030] env[64020]: DEBUG nova.virt.hardware [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1067.560901] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cb2b297-48c0-47c0-a256-f20712377f46 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1067.569184] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebf4baec-75e6-49fc-85c5-1395b4898cc4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1067.584027] env[64020]: ERROR nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 10ad0546-c51a-459e-9f38-34479abd3eb2, please check neutron logs for more information. [ 1067.584027] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Traceback (most recent call last): [ 1067.584027] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1067.584027] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] yield resources [ 1067.584027] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1067.584027] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] self.driver.spawn(context, instance, image_meta, [ 1067.584027] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1067.584027] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1067.584027] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1067.584027] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] vm_ref = self.build_virtual_machine(instance, [ 1067.584027] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] vif_infos = vmwarevif.get_vif_info(self._session, [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] for vif in network_info: [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] return self._sync_wrapper(fn, *args, **kwargs) [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] self.wait() [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] self[:] = self._gt.wait() [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] return self._exit_event.wait() [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1067.584740] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] current.throw(*self._exc) [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] result = function(*args, **kwargs) [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] return func(*args, **kwargs) [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] raise e [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] nwinfo = self.network_api.allocate_for_instance( [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] created_port_ids = self._update_ports_for_instance( [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] with excutils.save_and_reraise_exception(): [ 1067.585545] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1067.586152] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] self.force_reraise() [ 1067.586152] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1067.586152] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] raise self.value [ 1067.586152] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1067.586152] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] updated_port = self._update_port( [ 1067.586152] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1067.586152] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] _ensure_no_port_binding_failure(port) [ 1067.586152] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1067.586152] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] raise exception.PortBindingFailed(port_id=port['id']) [ 1067.586152] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] nova.exception.PortBindingFailed: Binding failed for port 10ad0546-c51a-459e-9f38-34479abd3eb2, please check neutron logs for more information. [ 1067.586152] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] [ 1067.586152] env[64020]: INFO nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Terminating instance [ 1067.586727] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Acquiring lock "refresh_cache-dea85e5f-ed81-41d2-bcea-906ba68c132b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1067.661308] env[64020]: DEBUG nova.network.neutron [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1067.731455] env[64020]: DEBUG nova.network.neutron [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1067.731975] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg c764b45274c149a88e8d76456e322b8e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1067.739596] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c764b45274c149a88e8d76456e322b8e [ 1068.021696] env[64020]: DEBUG oslo_concurrency.lockutils [req-ef18949c-96a9-4bd6-87a8-2629423e0679 req-46f6b411-107c-4118-913f-d0b71f7f32b6 service nova] Releasing lock "refresh_cache-dea85e5f-ed81-41d2-bcea-906ba68c132b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1068.022186] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Acquired lock "refresh_cache-dea85e5f-ed81-41d2-bcea-906ba68c132b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1068.022317] env[64020]: DEBUG nova.network.neutron [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1068.022753] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg bc43506ca9ea40c6b8d52dbb7197b863 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1068.038658] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc43506ca9ea40c6b8d52dbb7197b863 [ 1068.234614] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Releasing lock "refresh_cache-b46eab3d-f824-419d-822a-789618584907" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1068.234864] env[64020]: DEBUG nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1068.235050] env[64020]: DEBUG nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1068.235217] env[64020]: DEBUG nova.network.neutron [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1068.249785] env[64020]: DEBUG nova.network.neutron [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1068.250351] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg d05ec86bf22943f9a11eee2de4e50d16 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1068.257705] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d05ec86bf22943f9a11eee2de4e50d16 [ 1068.546852] env[64020]: DEBUG nova.network.neutron [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1068.632717] env[64020]: DEBUG nova.network.neutron [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1068.633235] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 7abf4680d9f54d08a92c301cedfc37b3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1068.641801] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7abf4680d9f54d08a92c301cedfc37b3 [ 1068.752602] env[64020]: DEBUG nova.network.neutron [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1068.753065] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 466417149caf4acdb7f9744325098207 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1068.763147] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 466417149caf4acdb7f9744325098207 [ 1068.958827] env[64020]: DEBUG nova.compute.manager [req-fa1d2599-6a9f-4f7b-9914-6c893a66bfa0 req-71280d64-74b0-471f-b7ab-364f02b20c7f service nova] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Received event network-vif-deleted-10ad0546-c51a-459e-9f38-34479abd3eb2 {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1069.135634] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Releasing lock "refresh_cache-dea85e5f-ed81-41d2-bcea-906ba68c132b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1069.136069] env[64020]: DEBUG nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1069.136308] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1069.136716] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9a7cdbbb-af91-4e7b-8437-ce876cdfaad6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1069.146221] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e8b0a224-0870-4e9d-b73c-15fdaeb24a0e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1069.167771] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance dea85e5f-ed81-41d2-bcea-906ba68c132b could not be found. [ 1069.167984] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1069.168188] env[64020]: INFO nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1069.168424] env[64020]: DEBUG oslo.service.loopingcall [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1069.168643] env[64020]: DEBUG nova.compute.manager [-] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1069.168737] env[64020]: DEBUG nova.network.neutron [-] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1069.186145] env[64020]: DEBUG nova.network.neutron [-] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1069.186638] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5701dfd8fd04455780c6c7db28b88d9f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1069.193365] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5701dfd8fd04455780c6c7db28b88d9f [ 1069.255394] env[64020]: INFO nova.compute.manager [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] [instance: b46eab3d-f824-419d-822a-789618584907] Took 1.02 seconds to deallocate network for instance. [ 1069.257219] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 494b025c3dd540c09bfc68b7b0eaab85 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1069.294377] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 494b025c3dd540c09bfc68b7b0eaab85 [ 1069.318018] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Acquiring lock "ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1069.319295] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Lock "ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1069.319295] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 5995be9e5c3541d9a958f9c4fe3211c0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1069.331214] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5995be9e5c3541d9a958f9c4fe3211c0 [ 1069.688904] env[64020]: DEBUG nova.network.neutron [-] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1069.689421] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 97e77c42cd7f42fe87d89c85e5be3230 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1069.698062] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 97e77c42cd7f42fe87d89c85e5be3230 [ 1069.761886] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg aef13b56015f4b7b989b84a318dbf7ef in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1069.802848] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aef13b56015f4b7b989b84a318dbf7ef [ 1069.821791] env[64020]: DEBUG nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1069.823538] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 5b946514dada4f16b2eddec6d8739fcf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1069.856627] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b946514dada4f16b2eddec6d8739fcf [ 1070.191756] env[64020]: INFO nova.compute.manager [-] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Took 1.02 seconds to deallocate network for instance. [ 1070.194565] env[64020]: DEBUG nova.compute.claims [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1070.194738] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1070.194955] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1070.196718] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg ac059ee451fd42dea7eff31f5fcd2ee1 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1070.247413] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ac059ee451fd42dea7eff31f5fcd2ee1 [ 1070.287132] env[64020]: INFO nova.scheduler.client.report [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Deleted allocations for instance b46eab3d-f824-419d-822a-789618584907 [ 1070.293204] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Expecting reply to msg 70b82f5217c2455fa4911215ba0bba65 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1070.304936] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 70b82f5217c2455fa4911215ba0bba65 [ 1070.340779] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1070.735222] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b81fdf60-9f37-4ce6-9760-13d24ae1f34e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1070.742331] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ee0c686-3dda-44d1-b96a-ba494956a9d9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1070.771943] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4da037a-a3c2-43a3-89e7-a6e4e499916a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1070.778799] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-364f12d5-6d18-4e5b-bd26-913efddc1b34 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1070.792444] env[64020]: DEBUG nova.compute.provider_tree [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1070.792963] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 43a5509a82834eb4b6797f45c420102d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1070.797503] env[64020]: DEBUG oslo_concurrency.lockutils [None req-20332e1a-fa9c-4b64-9cee-ca1f3e068967 tempest-DeleteServersTestJSON-2050988903 tempest-DeleteServersTestJSON-2050988903-project-member] Lock "b46eab3d-f824-419d-822a-789618584907" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 15.790s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1070.801604] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43a5509a82834eb4b6797f45c420102d [ 1071.298787] env[64020]: DEBUG nova.scheduler.client.report [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1071.298787] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg bec1146438d945a1b270117f60cc3a6e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1071.309543] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bec1146438d945a1b270117f60cc3a6e [ 1071.804105] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.606s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1071.804105] env[64020]: ERROR nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 10ad0546-c51a-459e-9f38-34479abd3eb2, please check neutron logs for more information. [ 1071.804105] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Traceback (most recent call last): [ 1071.804105] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1071.804105] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] self.driver.spawn(context, instance, image_meta, [ 1071.804105] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1071.804105] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1071.804105] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1071.804105] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] vm_ref = self.build_virtual_machine(instance, [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] vif_infos = vmwarevif.get_vif_info(self._session, [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] for vif in network_info: [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] return self._sync_wrapper(fn, *args, **kwargs) [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] self.wait() [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] self[:] = self._gt.wait() [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] return self._exit_event.wait() [ 1071.804658] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] current.throw(*self._exc) [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] result = function(*args, **kwargs) [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] return func(*args, **kwargs) [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] raise e [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] nwinfo = self.network_api.allocate_for_instance( [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] created_port_ids = self._update_ports_for_instance( [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1071.805032] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] with excutils.save_and_reraise_exception(): [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] self.force_reraise() [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] raise self.value [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] updated_port = self._update_port( [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] _ensure_no_port_binding_failure(port) [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] raise exception.PortBindingFailed(port_id=port['id']) [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] nova.exception.PortBindingFailed: Binding failed for port 10ad0546-c51a-459e-9f38-34479abd3eb2, please check neutron logs for more information. [ 1071.805414] env[64020]: ERROR nova.compute.manager [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] [ 1071.805773] env[64020]: DEBUG nova.compute.utils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Binding failed for port 10ad0546-c51a-459e-9f38-34479abd3eb2, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1071.806158] env[64020]: DEBUG nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Build of instance dea85e5f-ed81-41d2-bcea-906ba68c132b was re-scheduled: Binding failed for port 10ad0546-c51a-459e-9f38-34479abd3eb2, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1071.806634] env[64020]: DEBUG nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1071.806920] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Acquiring lock "refresh_cache-dea85e5f-ed81-41d2-bcea-906ba68c132b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1071.807129] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Acquired lock "refresh_cache-dea85e5f-ed81-41d2-bcea-906ba68c132b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1071.807341] env[64020]: DEBUG nova.network.neutron [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1071.807819] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 9310172861b04d11a614343acd64f954 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1071.810350] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 1.468s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1071.810350] env[64020]: INFO nova.compute.claims [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1071.811810] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 6a8a7c41bf784ea1a2a1e23fbdfb3bb2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1071.814271] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9310172861b04d11a614343acd64f954 [ 1071.848098] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a8a7c41bf784ea1a2a1e23fbdfb3bb2 [ 1072.317157] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 068e94bcca36405aa31a5095dd65b241 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1072.326407] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 068e94bcca36405aa31a5095dd65b241 [ 1072.332853] env[64020]: DEBUG nova.network.neutron [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1072.415174] env[64020]: DEBUG nova.network.neutron [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1072.415929] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 1e87de6af63746cfad972772672c06c7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1072.426061] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e87de6af63746cfad972772672c06c7 [ 1072.863212] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52551708-56d9-4164-a14b-baf0d8e0573c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1072.870469] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35f99dad-e263-44e2-9cbf-b26977d55d82 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1072.899995] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b953842a-f30f-4c8b-a4db-d4c408d9da48 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1072.907310] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d01ad180-aa55-4376-b718-f93e22a8842a {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1072.923621] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Releasing lock "refresh_cache-dea85e5f-ed81-41d2-bcea-906ba68c132b" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1072.923846] env[64020]: DEBUG nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1072.924036] env[64020]: DEBUG nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1072.924209] env[64020]: DEBUG nova.network.neutron [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1072.926551] env[64020]: DEBUG nova.compute.provider_tree [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1072.927187] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 1226e39c3fc8439c8f859db48f978601 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1072.935539] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1226e39c3fc8439c8f859db48f978601 [ 1072.942506] env[64020]: DEBUG nova.network.neutron [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1072.943285] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 4cbb6eb8699e4d989ebada995a37ef00 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1072.950426] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4cbb6eb8699e4d989ebada995a37ef00 [ 1073.429880] env[64020]: DEBUG nova.scheduler.client.report [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1073.432285] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 5278964c30e648968913350dcd4f4d9e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1073.443844] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5278964c30e648968913350dcd4f4d9e [ 1073.445503] env[64020]: DEBUG nova.network.neutron [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1073.445950] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg ddc2674b699145a5baeb19b8351ab441 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1073.455080] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ddc2674b699145a5baeb19b8351ab441 [ 1073.934759] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.126s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1073.935167] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 8a7da8eb13664b61afd06c659a45c8ae in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1073.948506] env[64020]: INFO nova.compute.manager [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] [instance: dea85e5f-ed81-41d2-bcea-906ba68c132b] Took 1.02 seconds to deallocate network for instance. [ 1073.950342] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 4fed433526924b4e8dc5b7c3fc8a65fa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1073.989054] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a7da8eb13664b61afd06c659a45c8ae [ 1073.993914] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4fed433526924b4e8dc5b7c3fc8a65fa [ 1074.437924] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Acquiring lock "53086c6e-37ef-4b96-a41f-93692ad4188e" by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1074.438225] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Lock "53086c6e-37ef-4b96-a41f-93692ad4188e" acquired by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1074.438691] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 712ba6f7457e44a0aa9790aa8ee68133 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1074.443870] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 712ba6f7457e44a0aa9790aa8ee68133 [ 1074.454328] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg 64d68bd3638944b3bbb82c657e7f92ef in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1074.496432] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64d68bd3638944b3bbb82c657e7f92ef [ 1074.952163] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Lock "53086c6e-37ef-4b96-a41f-93692ad4188e" "released" by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" :: held 0.503s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1074.952163] env[64020]: DEBUG nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1074.952163] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 3212fff59e6045738ff268589b747851 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1074.975203] env[64020]: INFO nova.scheduler.client.report [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Deleted allocations for instance dea85e5f-ed81-41d2-bcea-906ba68c132b [ 1074.981646] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Expecting reply to msg e8ff807b71d14c238de925f48c2f98e4 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1074.983029] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3212fff59e6045738ff268589b747851 [ 1075.007677] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e8ff807b71d14c238de925f48c2f98e4 [ 1075.204199] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Acquiring lock "79c9bea0-8c45-470b-87db-08264f01b553" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1075.204585] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Lock "79c9bea0-8c45-470b-87db-08264f01b553" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1075.205085] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 25a84423e5744674855b5732a1263106 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1075.213017] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 25a84423e5744674855b5732a1263106 [ 1075.454941] env[64020]: DEBUG nova.compute.utils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1075.455597] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 7c269d778df04ad3ae1521712230d8d8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1075.456630] env[64020]: DEBUG nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1075.456796] env[64020]: DEBUG nova.network.neutron [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1075.466063] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c269d778df04ad3ae1521712230d8d8 [ 1075.483336] env[64020]: DEBUG oslo_concurrency.lockutils [None req-a748d4a9-d995-40a3-90e6-625e51623b37 tempest-ServersNegativeTestMultiTenantJSON-630656320 tempest-ServersNegativeTestMultiTenantJSON-630656320-project-member] Lock "dea85e5f-ed81-41d2-bcea-906ba68c132b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 13.108s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1075.494280] env[64020]: DEBUG nova.policy [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a1dd06955a8a40e0993e7d5653cb5293', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '42d32543187f4384bd2a4a0236f188f5', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1075.709822] env[64020]: DEBUG nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Starting instance... {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1075.709822] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg b8a070922e6a49fe8e24c40f64c188f5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1075.751374] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8a070922e6a49fe8e24c40f64c188f5 [ 1075.774051] env[64020]: DEBUG nova.network.neutron [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Successfully created port: 32b22b67-489e-47dd-81e6-8ab7c2821d2b {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1075.962711] env[64020]: DEBUG nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1075.962711] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 99a0bdab1b2347b2906ae2dcc45abbaa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1075.993543] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99a0bdab1b2347b2906ae2dcc45abbaa [ 1076.228488] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1076.228924] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1076.230636] env[64020]: INFO nova.compute.claims [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1076.232444] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 73c4b290d58d432eaa693d3103c58cd3 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1076.269567] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 73c4b290d58d432eaa693d3103c58cd3 [ 1076.468094] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 5091c1ab66ca4f8dbba14d633414a3f0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1076.477502] env[64020]: DEBUG nova.compute.manager [req-51131543-f776-4bd0-b5ea-b655a3de642a req-22ef5c8d-fe72-4b60-9bc3-ef1f10d33d63 service nova] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Received event network-changed-32b22b67-489e-47dd-81e6-8ab7c2821d2b {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1076.477841] env[64020]: DEBUG nova.compute.manager [req-51131543-f776-4bd0-b5ea-b655a3de642a req-22ef5c8d-fe72-4b60-9bc3-ef1f10d33d63 service nova] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Refreshing instance network info cache due to event network-changed-32b22b67-489e-47dd-81e6-8ab7c2821d2b. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1076.478319] env[64020]: DEBUG oslo_concurrency.lockutils [req-51131543-f776-4bd0-b5ea-b655a3de642a req-22ef5c8d-fe72-4b60-9bc3-ef1f10d33d63 service nova] Acquiring lock "refresh_cache-ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1076.478897] env[64020]: DEBUG oslo_concurrency.lockutils [req-51131543-f776-4bd0-b5ea-b655a3de642a req-22ef5c8d-fe72-4b60-9bc3-ef1f10d33d63 service nova] Acquired lock "refresh_cache-ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1076.479176] env[64020]: DEBUG nova.network.neutron [req-51131543-f776-4bd0-b5ea-b655a3de642a req-22ef5c8d-fe72-4b60-9bc3-ef1f10d33d63 service nova] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Refreshing network info cache for port 32b22b67-489e-47dd-81e6-8ab7c2821d2b {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1076.479693] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-51131543-f776-4bd0-b5ea-b655a3de642a req-22ef5c8d-fe72-4b60-9bc3-ef1f10d33d63 service nova] Expecting reply to msg 5aa6afab1fad4ef78d52f0f44df9ccbb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1076.499687] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5aa6afab1fad4ef78d52f0f44df9ccbb [ 1076.510179] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5091c1ab66ca4f8dbba14d633414a3f0 [ 1076.608428] env[64020]: ERROR nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 32b22b67-489e-47dd-81e6-8ab7c2821d2b, please check neutron logs for more information. [ 1076.608428] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1076.608428] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1076.608428] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1076.608428] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1076.608428] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1076.608428] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1076.608428] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1076.608428] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1076.608428] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1076.608428] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1076.608428] env[64020]: ERROR nova.compute.manager raise self.value [ 1076.608428] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1076.608428] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1076.608428] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1076.608428] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1076.609109] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1076.609109] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1076.609109] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 32b22b67-489e-47dd-81e6-8ab7c2821d2b, please check neutron logs for more information. [ 1076.609109] env[64020]: ERROR nova.compute.manager [ 1076.609588] env[64020]: Traceback (most recent call last): [ 1076.609712] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1076.609712] env[64020]: listener.cb(fileno) [ 1076.609810] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1076.609810] env[64020]: result = function(*args, **kwargs) [ 1076.609896] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1076.609896] env[64020]: return func(*args, **kwargs) [ 1076.609986] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1076.609986] env[64020]: raise e [ 1076.610070] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1076.610070] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1076.610155] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1076.610155] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1076.610244] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1076.610244] env[64020]: with excutils.save_and_reraise_exception(): [ 1076.610323] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1076.610323] env[64020]: self.force_reraise() [ 1076.610407] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1076.610407] env[64020]: raise self.value [ 1076.612018] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1076.612018] env[64020]: updated_port = self._update_port( [ 1076.612123] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1076.612123] env[64020]: _ensure_no_port_binding_failure(port) [ 1076.612204] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1076.612204] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1076.612326] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 32b22b67-489e-47dd-81e6-8ab7c2821d2b, please check neutron logs for more information. [ 1076.612383] env[64020]: Removing descriptor: 16 [ 1076.747334] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 184ed647cb114d0cbf04046f12ac19bf in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1076.754826] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 184ed647cb114d0cbf04046f12ac19bf [ 1076.972623] env[64020]: DEBUG nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1077.000099] env[64020]: DEBUG nova.virt.hardware [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1077.001479] env[64020]: DEBUG nova.virt.hardware [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1077.001785] env[64020]: DEBUG nova.virt.hardware [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1077.002088] env[64020]: DEBUG nova.virt.hardware [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1077.002945] env[64020]: DEBUG nova.virt.hardware [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1077.003239] env[64020]: DEBUG nova.virt.hardware [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1077.003576] env[64020]: DEBUG nova.virt.hardware [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1077.004156] env[64020]: DEBUG nova.virt.hardware [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1077.004451] env[64020]: DEBUG nova.virt.hardware [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1077.004743] env[64020]: DEBUG nova.virt.hardware [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1077.005029] env[64020]: DEBUG nova.virt.hardware [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1077.006896] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa8780bd-2718-43b2-ad87-d25a701522a7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1077.019886] env[64020]: DEBUG nova.network.neutron [req-51131543-f776-4bd0-b5ea-b655a3de642a req-22ef5c8d-fe72-4b60-9bc3-ef1f10d33d63 service nova] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1077.026656] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a70ac8b0-97ea-4da7-9660-20c9fd79dc96 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1077.041460] env[64020]: ERROR nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 32b22b67-489e-47dd-81e6-8ab7c2821d2b, please check neutron logs for more information. [ 1077.041460] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Traceback (most recent call last): [ 1077.041460] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1077.041460] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] yield resources [ 1077.041460] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1077.041460] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] self.driver.spawn(context, instance, image_meta, [ 1077.041460] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1077.041460] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1077.041460] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1077.041460] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] vm_ref = self.build_virtual_machine(instance, [ 1077.041460] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] vif_infos = vmwarevif.get_vif_info(self._session, [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] for vif in network_info: [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] return self._sync_wrapper(fn, *args, **kwargs) [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] self.wait() [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] self[:] = self._gt.wait() [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] return self._exit_event.wait() [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1077.041851] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] current.throw(*self._exc) [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] result = function(*args, **kwargs) [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] return func(*args, **kwargs) [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] raise e [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] nwinfo = self.network_api.allocate_for_instance( [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] created_port_ids = self._update_ports_for_instance( [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] with excutils.save_and_reraise_exception(): [ 1077.042268] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1077.042696] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] self.force_reraise() [ 1077.042696] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1077.042696] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] raise self.value [ 1077.042696] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1077.042696] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] updated_port = self._update_port( [ 1077.042696] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1077.042696] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] _ensure_no_port_binding_failure(port) [ 1077.042696] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1077.042696] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] raise exception.PortBindingFailed(port_id=port['id']) [ 1077.042696] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] nova.exception.PortBindingFailed: Binding failed for port 32b22b67-489e-47dd-81e6-8ab7c2821d2b, please check neutron logs for more information. [ 1077.042696] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] [ 1077.042696] env[64020]: INFO nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Terminating instance [ 1077.043798] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Acquiring lock "refresh_cache-ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1077.092896] env[64020]: DEBUG nova.network.neutron [req-51131543-f776-4bd0-b5ea-b655a3de642a req-22ef5c8d-fe72-4b60-9bc3-ef1f10d33d63 service nova] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1077.093511] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-51131543-f776-4bd0-b5ea-b655a3de642a req-22ef5c8d-fe72-4b60-9bc3-ef1f10d33d63 service nova] Expecting reply to msg 86e4214380d043dfa59d194c7bbda635 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1077.103037] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86e4214380d043dfa59d194c7bbda635 [ 1077.317438] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81053f96-c91b-4410-bec4-a204738ddd01 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1077.326936] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee4443aa-f100-4ee9-bb72-87fda08ba1be {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1077.357398] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0194ff78-98ae-43f2-99d8-441d561a631e {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1077.364888] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c945e354-ff95-4474-90e5-e30fe7884ea4 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1077.378767] env[64020]: DEBUG nova.compute.provider_tree [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1077.379286] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 56e9939d6c134a45a84de077676fd636 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1077.386732] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 56e9939d6c134a45a84de077676fd636 [ 1077.599434] env[64020]: DEBUG oslo_concurrency.lockutils [req-51131543-f776-4bd0-b5ea-b655a3de642a req-22ef5c8d-fe72-4b60-9bc3-ef1f10d33d63 service nova] Releasing lock "refresh_cache-ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1077.599920] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Acquired lock "refresh_cache-ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1077.600120] env[64020]: DEBUG nova.network.neutron [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1077.600543] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 1a7824243f9e48a0911c1bf16971e15a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1077.608354] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a7824243f9e48a0911c1bf16971e15a [ 1077.882378] env[64020]: DEBUG nova.scheduler.client.report [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1077.884721] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 42f659060a874acda8a0f1c8def8b33d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1077.895798] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42f659060a874acda8a0f1c8def8b33d [ 1078.117447] env[64020]: DEBUG nova.network.neutron [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1078.188679] env[64020]: DEBUG nova.network.neutron [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1078.189234] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg eba29c9d52c64f388667144c449187d0 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1078.198055] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eba29c9d52c64f388667144c449187d0 [ 1078.386898] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.158s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1078.387440] env[64020]: DEBUG nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Start building networks asynchronously for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1078.389316] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 4ab9afff26a74af7a4aee4f75d41077e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1078.427004] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4ab9afff26a74af7a4aee4f75d41077e [ 1078.502814] env[64020]: DEBUG nova.compute.manager [req-7a3173ba-f2e6-42cc-8784-27092d0adc1c req-9c4ae31b-4618-489e-8dfc-652db04c2a4b service nova] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Received event network-vif-deleted-32b22b67-489e-47dd-81e6-8ab7c2821d2b {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1078.691051] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Releasing lock "refresh_cache-ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1078.691522] env[64020]: DEBUG nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1078.691714] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1078.692039] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9fcf104c-f007-43f8-ac18-78311953ea59 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1078.701655] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3a6ba7fd-dfe9-4b57-a23a-a364af7607e8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1078.722933] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ebaab6d8-7c21-4759-b8cc-28f6e963fd1d could not be found. [ 1078.723428] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1078.723844] env[64020]: INFO nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1078.724426] env[64020]: DEBUG oslo.service.loopingcall [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1078.724925] env[64020]: DEBUG nova.compute.manager [-] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1078.725172] env[64020]: DEBUG nova.network.neutron [-] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1078.739471] env[64020]: DEBUG nova.network.neutron [-] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1078.740041] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7744a747d0b847918f504f7540f53aaa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1078.748094] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7744a747d0b847918f504f7540f53aaa [ 1078.893157] env[64020]: DEBUG nova.compute.utils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Using /dev/sd instead of None {{(pid=64020) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1078.893777] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 6ba2badda4944ce483e8bc9f5617303f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1078.894683] env[64020]: DEBUG nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Allocating IP information in the background. {{(pid=64020) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1078.894845] env[64020]: DEBUG nova.network.neutron [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] allocate_for_instance() {{(pid=64020) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1078.903678] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ba2badda4944ce483e8bc9f5617303f [ 1078.930945] env[64020]: DEBUG nova.policy [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '0ebbc96149de425a86849d8e65fb5713', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '46166314d94c42219f44cd4b1494afaa', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=64020) authorize /opt/stack/nova/nova/policy.py:203}} [ 1079.165428] env[64020]: DEBUG nova.network.neutron [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Successfully created port: 48dd7a6e-e5c8-435c-b668-015b1e3952fd {{(pid=64020) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1079.241875] env[64020]: DEBUG nova.network.neutron [-] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1079.242380] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3c1f9870634d4e2a857fcb5b2e27b4b6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1079.250446] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c1f9870634d4e2a857fcb5b2e27b4b6 [ 1079.398111] env[64020]: DEBUG nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Start building block device mappings for instance. {{(pid=64020) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1079.400013] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 11506e01184b4c6caea96273ffbd8865 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1079.434376] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 11506e01184b4c6caea96273ffbd8865 [ 1079.744185] env[64020]: INFO nova.compute.manager [-] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Took 1.02 seconds to deallocate network for instance. [ 1079.746336] env[64020]: DEBUG nova.compute.claims [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1079.746530] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1079.746782] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1079.748555] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 141ae0d55a064c41981d126ac61ad00e in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1079.778820] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 141ae0d55a064c41981d126ac61ad00e [ 1079.904901] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg f7030cd3c85b48e093b0288bee07ad04 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1079.934528] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7030cd3c85b48e093b0288bee07ad04 [ 1079.960275] env[64020]: ERROR nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 48dd7a6e-e5c8-435c-b668-015b1e3952fd, please check neutron logs for more information. [ 1079.960275] env[64020]: ERROR nova.compute.manager Traceback (most recent call last): [ 1079.960275] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1079.960275] env[64020]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1079.960275] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1079.960275] env[64020]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1079.960275] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1079.960275] env[64020]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1079.960275] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1079.960275] env[64020]: ERROR nova.compute.manager self.force_reraise() [ 1079.960275] env[64020]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1079.960275] env[64020]: ERROR nova.compute.manager raise self.value [ 1079.960275] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1079.960275] env[64020]: ERROR nova.compute.manager updated_port = self._update_port( [ 1079.960275] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1079.960275] env[64020]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1079.960825] env[64020]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1079.960825] env[64020]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1079.960825] env[64020]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 48dd7a6e-e5c8-435c-b668-015b1e3952fd, please check neutron logs for more information. [ 1079.960825] env[64020]: ERROR nova.compute.manager [ 1079.960825] env[64020]: Traceback (most recent call last): [ 1079.960825] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1079.960825] env[64020]: listener.cb(fileno) [ 1079.960825] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1079.960825] env[64020]: result = function(*args, **kwargs) [ 1079.960825] env[64020]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1079.960825] env[64020]: return func(*args, **kwargs) [ 1079.960825] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1079.960825] env[64020]: raise e [ 1079.960825] env[64020]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1079.960825] env[64020]: nwinfo = self.network_api.allocate_for_instance( [ 1079.960825] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1079.960825] env[64020]: created_port_ids = self._update_ports_for_instance( [ 1079.960825] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1079.960825] env[64020]: with excutils.save_and_reraise_exception(): [ 1079.960825] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1079.960825] env[64020]: self.force_reraise() [ 1079.960825] env[64020]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1079.960825] env[64020]: raise self.value [ 1079.960825] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1079.960825] env[64020]: updated_port = self._update_port( [ 1079.960825] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1079.960825] env[64020]: _ensure_no_port_binding_failure(port) [ 1079.960825] env[64020]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1079.960825] env[64020]: raise exception.PortBindingFailed(port_id=port['id']) [ 1079.961704] env[64020]: nova.exception.PortBindingFailed: Binding failed for port 48dd7a6e-e5c8-435c-b668-015b1e3952fd, please check neutron logs for more information. [ 1079.961704] env[64020]: Removing descriptor: 16 [ 1080.290105] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7dcd98a8-81aa-4626-9a5d-d28acfdfc25d {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.297707] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92bd6ea6-a6c6-4144-91b9-6a838210e0f9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.325744] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65bbe525-da4f-4654-b6b7-140a144ba60c {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.331965] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b5e67d8-f395-4fb3-9f17-95f037d510d8 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.343941] env[64020]: DEBUG nova.compute.provider_tree [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1080.344417] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 005f6f2ed8ec482a90dcbee16a97b21c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1080.351262] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 005f6f2ed8ec482a90dcbee16a97b21c [ 1080.407020] env[64020]: DEBUG nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Start spawning the instance on the hypervisor. {{(pid=64020) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1080.432433] env[64020]: DEBUG nova.virt.hardware [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-22T23:24:17Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-22T23:23:59Z,direct_url=,disk_format='vmdk',id=9fb27d9a-1b35-489a-a2e9-d7ab6f1e6234,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='f746be43fab941fbb56c7c62b819fd52',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-22T23:24:00Z,virtual_size=,visibility=), allow threads: False {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1080.432690] env[64020]: DEBUG nova.virt.hardware [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Flavor limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1080.432846] env[64020]: DEBUG nova.virt.hardware [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Image limits 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1080.433023] env[64020]: DEBUG nova.virt.hardware [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Flavor pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1080.433164] env[64020]: DEBUG nova.virt.hardware [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Image pref 0:0:0 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1080.433305] env[64020]: DEBUG nova.virt.hardware [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=64020) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1080.433924] env[64020]: DEBUG nova.virt.hardware [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1080.433924] env[64020]: DEBUG nova.virt.hardware [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1080.433924] env[64020]: DEBUG nova.virt.hardware [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Got 1 possible topologies {{(pid=64020) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1080.434149] env[64020]: DEBUG nova.virt.hardware [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1080.434149] env[64020]: DEBUG nova.virt.hardware [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=64020) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1080.434909] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28149070-5b35-4823-8c57-b1e1ae34d992 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.442189] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32f2c876-b256-4ea7-b42b-3affd0854fee {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.454973] env[64020]: ERROR nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 48dd7a6e-e5c8-435c-b668-015b1e3952fd, please check neutron logs for more information. [ 1080.454973] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Traceback (most recent call last): [ 1080.454973] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1080.454973] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] yield resources [ 1080.454973] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1080.454973] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] self.driver.spawn(context, instance, image_meta, [ 1080.454973] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1080.454973] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1080.454973] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1080.454973] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] vm_ref = self.build_virtual_machine(instance, [ 1080.454973] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] vif_infos = vmwarevif.get_vif_info(self._session, [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] for vif in network_info: [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] return self._sync_wrapper(fn, *args, **kwargs) [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] self.wait() [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] self[:] = self._gt.wait() [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] return self._exit_event.wait() [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1080.455616] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] current.throw(*self._exc) [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] result = function(*args, **kwargs) [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] return func(*args, **kwargs) [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] raise e [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] nwinfo = self.network_api.allocate_for_instance( [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] created_port_ids = self._update_ports_for_instance( [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] with excutils.save_and_reraise_exception(): [ 1080.456256] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1080.456978] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] self.force_reraise() [ 1080.456978] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1080.456978] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] raise self.value [ 1080.456978] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1080.456978] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] updated_port = self._update_port( [ 1080.456978] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1080.456978] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] _ensure_no_port_binding_failure(port) [ 1080.456978] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1080.456978] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] raise exception.PortBindingFailed(port_id=port['id']) [ 1080.456978] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] nova.exception.PortBindingFailed: Binding failed for port 48dd7a6e-e5c8-435c-b668-015b1e3952fd, please check neutron logs for more information. [ 1080.456978] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] [ 1080.456978] env[64020]: INFO nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Terminating instance [ 1080.458092] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Acquiring lock "refresh_cache-79c9bea0-8c45-470b-87db-08264f01b553" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1080.458267] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Acquired lock "refresh_cache-79c9bea0-8c45-470b-87db-08264f01b553" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1080.458713] env[64020]: DEBUG nova.network.neutron [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1080.459439] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 4a54e25bec324af581d19a91823ea462 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1080.466512] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4a54e25bec324af581d19a91823ea462 [ 1080.526865] env[64020]: DEBUG nova.compute.manager [req-fb5f5acb-4bd9-4251-9189-d2cb6cad5b80 req-376eae42-179d-465f-8083-71d9ee337549 service nova] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Received event network-changed-48dd7a6e-e5c8-435c-b668-015b1e3952fd {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1080.527032] env[64020]: DEBUG nova.compute.manager [req-fb5f5acb-4bd9-4251-9189-d2cb6cad5b80 req-376eae42-179d-465f-8083-71d9ee337549 service nova] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Refreshing instance network info cache due to event network-changed-48dd7a6e-e5c8-435c-b668-015b1e3952fd. {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1080.527224] env[64020]: DEBUG oslo_concurrency.lockutils [req-fb5f5acb-4bd9-4251-9189-d2cb6cad5b80 req-376eae42-179d-465f-8083-71d9ee337549 service nova] Acquiring lock "refresh_cache-79c9bea0-8c45-470b-87db-08264f01b553" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1080.846512] env[64020]: DEBUG nova.scheduler.client.report [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1080.848857] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 45969d61775d4fbeb922804608707a8c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1080.878889] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 45969d61775d4fbeb922804608707a8c [ 1080.975734] env[64020]: DEBUG nova.network.neutron [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1081.062740] env[64020]: DEBUG nova.network.neutron [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1081.063232] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg eea0ba67e06548d193e85e0fa128f7aa in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1081.071011] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eea0ba67e06548d193e85e0fa128f7aa [ 1081.351855] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.605s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.352496] env[64020]: ERROR nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 32b22b67-489e-47dd-81e6-8ab7c2821d2b, please check neutron logs for more information. [ 1081.352496] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Traceback (most recent call last): [ 1081.352496] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1081.352496] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] self.driver.spawn(context, instance, image_meta, [ 1081.352496] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1081.352496] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1081.352496] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1081.352496] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] vm_ref = self.build_virtual_machine(instance, [ 1081.352496] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1081.352496] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] vif_infos = vmwarevif.get_vif_info(self._session, [ 1081.352496] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] for vif in network_info: [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] return self._sync_wrapper(fn, *args, **kwargs) [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] self.wait() [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] self[:] = self._gt.wait() [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] return self._exit_event.wait() [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] current.throw(*self._exc) [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1081.352765] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] result = function(*args, **kwargs) [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] return func(*args, **kwargs) [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] raise e [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] nwinfo = self.network_api.allocate_for_instance( [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] created_port_ids = self._update_ports_for_instance( [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] with excutils.save_and_reraise_exception(): [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] self.force_reraise() [ 1081.353034] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1081.353366] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] raise self.value [ 1081.353366] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1081.353366] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] updated_port = self._update_port( [ 1081.353366] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1081.353366] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] _ensure_no_port_binding_failure(port) [ 1081.353366] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1081.353366] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] raise exception.PortBindingFailed(port_id=port['id']) [ 1081.353366] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] nova.exception.PortBindingFailed: Binding failed for port 32b22b67-489e-47dd-81e6-8ab7c2821d2b, please check neutron logs for more information. [ 1081.353366] env[64020]: ERROR nova.compute.manager [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] [ 1081.353366] env[64020]: DEBUG nova.compute.utils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Binding failed for port 32b22b67-489e-47dd-81e6-8ab7c2821d2b, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1081.354670] env[64020]: DEBUG nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Build of instance ebaab6d8-7c21-4759-b8cc-28f6e963fd1d was re-scheduled: Binding failed for port 32b22b67-489e-47dd-81e6-8ab7c2821d2b, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1081.355050] env[64020]: DEBUG nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1081.355270] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Acquiring lock "refresh_cache-ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1081.355416] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Acquired lock "refresh_cache-ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1081.355570] env[64020]: DEBUG nova.network.neutron [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1081.355965] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 925f342de77c4ec1991e62cd16f35379 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1081.363079] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 925f342de77c4ec1991e62cd16f35379 [ 1081.565204] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Releasing lock "refresh_cache-79c9bea0-8c45-470b-87db-08264f01b553" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1081.565616] env[64020]: DEBUG nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Start destroying the instance on the hypervisor. {{(pid=64020) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1081.565847] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Destroying instance {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1081.566107] env[64020]: DEBUG oslo_concurrency.lockutils [req-fb5f5acb-4bd9-4251-9189-d2cb6cad5b80 req-376eae42-179d-465f-8083-71d9ee337549 service nova] Acquired lock "refresh_cache-79c9bea0-8c45-470b-87db-08264f01b553" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1081.566275] env[64020]: DEBUG nova.network.neutron [req-fb5f5acb-4bd9-4251-9189-d2cb6cad5b80 req-376eae42-179d-465f-8083-71d9ee337549 service nova] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Refreshing network info cache for port 48dd7a6e-e5c8-435c-b668-015b1e3952fd {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1081.566895] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-fb5f5acb-4bd9-4251-9189-d2cb6cad5b80 req-376eae42-179d-465f-8083-71d9ee337549 service nova] Expecting reply to msg da99621af96a492ca25a76fbe7ac4fb7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1081.572404] env[64020]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-be5e3448-5b2d-4560-85a8-24215a908969 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1081.575051] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg da99621af96a492ca25a76fbe7ac4fb7 [ 1081.577760] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af5108ce-fedb-47ac-a0db-99149ee954ce {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1081.598837] env[64020]: WARNING nova.virt.vmwareapi.vmops [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 79c9bea0-8c45-470b-87db-08264f01b553 could not be found. [ 1081.599103] env[64020]: DEBUG nova.virt.vmwareapi.vmops [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Instance destroyed {{(pid=64020) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1081.599305] env[64020]: INFO nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1081.599547] env[64020]: DEBUG oslo.service.loopingcall [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=64020) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1081.599766] env[64020]: DEBUG nova.compute.manager [-] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1081.599859] env[64020]: DEBUG nova.network.neutron [-] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1081.614005] env[64020]: DEBUG nova.network.neutron [-] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1081.614484] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bc5901b6b9394be59647aa2ca8163e64 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1081.621339] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc5901b6b9394be59647aa2ca8163e64 [ 1081.873255] env[64020]: DEBUG nova.network.neutron [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1081.943029] env[64020]: DEBUG nova.network.neutron [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1081.943550] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 52961d3c367c43f38e4ef5f17f91df22 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1081.951895] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52961d3c367c43f38e4ef5f17f91df22 [ 1082.091359] env[64020]: DEBUG nova.network.neutron [req-fb5f5acb-4bd9-4251-9189-d2cb6cad5b80 req-376eae42-179d-465f-8083-71d9ee337549 service nova] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1082.116507] env[64020]: DEBUG nova.network.neutron [-] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1082.116965] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 632da1d359154bf39868bef8c2ca5bf8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1082.125831] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 632da1d359154bf39868bef8c2ca5bf8 [ 1082.164437] env[64020]: DEBUG nova.network.neutron [req-fb5f5acb-4bd9-4251-9189-d2cb6cad5b80 req-376eae42-179d-465f-8083-71d9ee337549 service nova] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1082.164931] env[64020]: INFO oslo_messaging._drivers.amqpdriver [req-fb5f5acb-4bd9-4251-9189-d2cb6cad5b80 req-376eae42-179d-465f-8083-71d9ee337549 service nova] Expecting reply to msg 6847810597bf42e2b594aad2ea956ef8 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1082.172785] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6847810597bf42e2b594aad2ea956ef8 [ 1082.445479] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Releasing lock "refresh_cache-ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1082.445800] env[64020]: DEBUG nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1082.445853] env[64020]: DEBUG nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1082.446008] env[64020]: DEBUG nova.network.neutron [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1082.461697] env[64020]: DEBUG nova.network.neutron [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1082.462280] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 6dfbc53345714878beb71d78719fdf89 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1082.468713] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6dfbc53345714878beb71d78719fdf89 [ 1082.619099] env[64020]: INFO nova.compute.manager [-] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Took 1.02 seconds to deallocate network for instance. [ 1082.621571] env[64020]: DEBUG nova.compute.claims [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Aborting claim: {{(pid=64020) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1082.621707] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1082.621921] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1082.623753] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg a982cf1fd37c4581b83a90afa15f04fb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1082.655919] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a982cf1fd37c4581b83a90afa15f04fb [ 1082.666483] env[64020]: DEBUG oslo_concurrency.lockutils [req-fb5f5acb-4bd9-4251-9189-d2cb6cad5b80 req-376eae42-179d-465f-8083-71d9ee337549 service nova] Releasing lock "refresh_cache-79c9bea0-8c45-470b-87db-08264f01b553" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1082.666723] env[64020]: DEBUG nova.compute.manager [req-fb5f5acb-4bd9-4251-9189-d2cb6cad5b80 req-376eae42-179d-465f-8083-71d9ee337549 service nova] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Received event network-vif-deleted-48dd7a6e-e5c8-435c-b668-015b1e3952fd {{(pid=64020) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1082.964870] env[64020]: DEBUG nova.network.neutron [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1082.965357] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 9ca932cd93e147e689a9a35c90698fb6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1082.974117] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ca932cd93e147e689a9a35c90698fb6 [ 1083.162632] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-266b4aaa-b998-49a0-9b53-00a1a2678eff {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1083.171246] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd7e8995-7b8c-446a-9266-7505f08482b9 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1083.199934] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c392c669-1501-4df2-a66e-a265fa1b56b6 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1083.206846] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10de21ad-4f77-42fc-ad0a-ba94778e0b06 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1083.219395] env[64020]: DEBUG nova.compute.provider_tree [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1083.219857] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg ea5707b11e324b44b85c179294ae842c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1083.226811] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ea5707b11e324b44b85c179294ae842c [ 1083.468267] env[64020]: INFO nova.compute.manager [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] [instance: ebaab6d8-7c21-4759-b8cc-28f6e963fd1d] Took 1.02 seconds to deallocate network for instance. [ 1083.470211] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 3a7757efa28f41c5976bca0f4103fcc5 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1083.500524] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3a7757efa28f41c5976bca0f4103fcc5 [ 1083.723027] env[64020]: DEBUG nova.scheduler.client.report [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1083.725392] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg dfb7decff6114b58900137f723ca9e5a in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1083.737512] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dfb7decff6114b58900137f723ca9e5a [ 1083.974680] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg 47e1d7ee6cc84efab80e1f6d25d73b13 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1084.007493] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47e1d7ee6cc84efab80e1f6d25d73b13 [ 1084.227809] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.606s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1084.228449] env[64020]: ERROR nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 48dd7a6e-e5c8-435c-b668-015b1e3952fd, please check neutron logs for more information. [ 1084.228449] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Traceback (most recent call last): [ 1084.228449] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1084.228449] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] self.driver.spawn(context, instance, image_meta, [ 1084.228449] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1084.228449] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1084.228449] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1084.228449] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] vm_ref = self.build_virtual_machine(instance, [ 1084.228449] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1084.228449] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] vif_infos = vmwarevif.get_vif_info(self._session, [ 1084.228449] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] for vif in network_info: [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] return self._sync_wrapper(fn, *args, **kwargs) [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] self.wait() [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] self[:] = self._gt.wait() [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] return self._exit_event.wait() [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] current.throw(*self._exc) [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1084.228792] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] result = function(*args, **kwargs) [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] return func(*args, **kwargs) [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] raise e [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] nwinfo = self.network_api.allocate_for_instance( [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] created_port_ids = self._update_ports_for_instance( [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] with excutils.save_and_reraise_exception(): [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] self.force_reraise() [ 1084.229106] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1084.229430] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] raise self.value [ 1084.229430] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1084.229430] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] updated_port = self._update_port( [ 1084.229430] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1084.229430] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] _ensure_no_port_binding_failure(port) [ 1084.229430] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1084.229430] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] raise exception.PortBindingFailed(port_id=port['id']) [ 1084.229430] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] nova.exception.PortBindingFailed: Binding failed for port 48dd7a6e-e5c8-435c-b668-015b1e3952fd, please check neutron logs for more information. [ 1084.229430] env[64020]: ERROR nova.compute.manager [instance: 79c9bea0-8c45-470b-87db-08264f01b553] [ 1084.229430] env[64020]: DEBUG nova.compute.utils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Binding failed for port 48dd7a6e-e5c8-435c-b668-015b1e3952fd, please check neutron logs for more information. {{(pid=64020) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1084.230764] env[64020]: DEBUG nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Build of instance 79c9bea0-8c45-470b-87db-08264f01b553 was re-scheduled: Binding failed for port 48dd7a6e-e5c8-435c-b668-015b1e3952fd, please check neutron logs for more information. {{(pid=64020) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1084.231150] env[64020]: DEBUG nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Unplugging VIFs for instance {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1084.231402] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Acquiring lock "refresh_cache-79c9bea0-8c45-470b-87db-08264f01b553" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1084.231549] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Acquired lock "refresh_cache-79c9bea0-8c45-470b-87db-08264f01b553" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1084.231705] env[64020]: DEBUG nova.network.neutron [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Building network info cache for instance {{(pid=64020) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1084.232134] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg ef2bdb8a0b9e4be9b3f3b39c2d4dad11 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1084.238510] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef2bdb8a0b9e4be9b3f3b39c2d4dad11 [ 1084.498490] env[64020]: INFO nova.scheduler.client.report [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Deleted allocations for instance ebaab6d8-7c21-4759-b8cc-28f6e963fd1d [ 1084.505006] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Expecting reply to msg e47b8b37fa4e42b69b693a4512b89001 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1084.518694] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e47b8b37fa4e42b69b693a4512b89001 [ 1084.750298] env[64020]: DEBUG nova.network.neutron [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1084.831109] env[64020]: DEBUG nova.network.neutron [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1084.831729] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 6715282da83c43d0a567216ce4394764 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1084.840188] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6715282da83c43d0a567216ce4394764 [ 1085.007448] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e465f2b7-260e-4a37-af1f-a92279e1b071 tempest-ServerGroupTestJSON-1248368617 tempest-ServerGroupTestJSON-1248368617-project-member] Lock "ebaab6d8-7c21-4759-b8cc-28f6e963fd1d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 15.689s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1085.334303] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Releasing lock "refresh_cache-79c9bea0-8c45-470b-87db-08264f01b553" {{(pid=64020) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1085.334590] env[64020]: DEBUG nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=64020) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1085.335025] env[64020]: DEBUG nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Deallocating network for instance {{(pid=64020) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1085.335025] env[64020]: DEBUG nova.network.neutron [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] deallocate_for_instance() {{(pid=64020) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1085.348893] env[64020]: DEBUG nova.network.neutron [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Instance cache missing network info. {{(pid=64020) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1085.349565] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 46fe0dad45744fb4bd3ea571a5c3f1f6 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1085.356697] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46fe0dad45744fb4bd3ea571a5c3f1f6 [ 1085.851874] env[64020]: DEBUG nova.network.neutron [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Updating instance_info_cache with network_info: [] {{(pid=64020) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1085.852510] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 7ffb6c9fd884436e97418d3d1dcab86d in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1085.861443] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ffb6c9fd884436e97418d3d1dcab86d [ 1086.354658] env[64020]: INFO nova.compute.manager [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] [instance: 79c9bea0-8c45-470b-87db-08264f01b553] Took 1.02 seconds to deallocate network for instance. [ 1086.356459] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 436e2668f23643f59438e37f8df6e41c in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1086.394048] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 436e2668f23643f59438e37f8df6e41c [ 1086.861031] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg 054e9725979f4f29ae276788beb68e60 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1086.889236] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 054e9725979f4f29ae276788beb68e60 [ 1087.384107] env[64020]: INFO nova.scheduler.client.report [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Deleted allocations for instance 79c9bea0-8c45-470b-87db-08264f01b553 [ 1087.390198] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Expecting reply to msg a2265a5676c340b698e0e104c5591076 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1087.403212] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2265a5676c340b698e0e104c5591076 [ 1087.892291] env[64020]: DEBUG oslo_concurrency.lockutils [None req-e532d1b7-6cd9-49b3-99b3-c818b5158312 tempest-ServersNegativeTestJSON-1167413370 tempest-ServersNegativeTestJSON-1167413370-project-member] Lock "79c9bea0-8c45-470b-87db-08264f01b553" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 12.688s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1098.202026] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1098.202471] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1098.202551] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Starting heal instance info cache {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1098.202609] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Rebuilding the list of instances to heal {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1098.203226] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 8c7ab0f64fa54b57b8b46b372bfbdf98 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1098.213138] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8c7ab0f64fa54b57b8b46b372bfbdf98 [ 1098.705085] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Didn't find any instances for network info cache update. {{(pid=64020) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1098.705324] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1098.705494] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1098.705643] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1098.705793] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1098.705938] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1098.706082] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1098.706208] env[64020]: DEBUG nova.compute.manager [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=64020) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1098.706350] env[64020]: DEBUG oslo_service.periodic_task [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Running periodic task ComputeManager.update_available_resource {{(pid=64020) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1098.706685] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 8c293f33bee743809ded4de1d7ad851f in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1098.716668] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8c293f33bee743809ded4de1d7ad851f [ 1099.211796] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1099.211796] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1099.211796] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1099.211796] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=64020) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1099.211796] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af3e0616-99d6-41a1-9b6f-aca2b0768d9b {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1099.219388] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ef6f690-0cc4-4d2c-8691-8c9ae2fb41f7 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1099.232827] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb1ad2e1-db1f-4806-a0b1-16c73788d883 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1099.238599] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-222062ec-f633-4cb2-9573-28ee7bd6b281 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1099.267126] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181780MB free_disk=120GB free_vcpus=48 pci_devices=None {{(pid=64020) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1099.267247] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1099.267411] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1099.268226] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg e54308d600574203adfc40646d1f3bb2 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1099.275900] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e54308d600574203adfc40646d1f3bb2 [ 1099.771037] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg c1a19bb7192d42be836f5b5183520955 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1099.781033] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c1a19bb7192d42be836f5b5183520955 [ 1100.291221] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1100.291497] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=64020) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1100.304066] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7390f8a-d5cc-48ac-8edd-02f9f16898bd {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1100.311440] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-745a47a7-f24b-44b1-a99a-b2a74d46c8e2 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1100.339299] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2fdc01e-f632-499a-9c1b-6948399bca39 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1100.345759] env[64020]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c62232b-bbd3-4cac-b967-95c566afd814 {{(pid=64020) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1100.358739] env[64020]: DEBUG nova.compute.provider_tree [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed in ProviderTree for provider: 092888c9-2221-4dfc-9104-eeeb335c764f {{(pid=64020) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1100.359172] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg e93eabcb623c4b57b36ebeabd212ffcb in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1100.368293] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e93eabcb623c4b57b36ebeabd212ffcb [ 1100.862165] env[64020]: DEBUG nova.scheduler.client.report [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Inventory has not changed for provider 092888c9-2221-4dfc-9104-eeeb335c764f based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 120, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=64020) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1100.864496] env[64020]: INFO oslo_messaging._drivers.amqpdriver [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Expecting reply to msg 862ddf6b5e0845389ed9626381342ce7 in queue reply_57893177120949e6a93cb88e15cd42b4 [ 1100.875878] env[64020]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 862ddf6b5e0845389ed9626381342ce7 [ 1101.367876] env[64020]: DEBUG nova.compute.resource_tracker [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=64020) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1101.368236] env[64020]: DEBUG oslo_concurrency.lockutils [None req-cd19f7e3-6315-44a9-8750-a815853513ce None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.101s {{(pid=64020) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}}